from util import get_std_lib, get_third_lib, get_buildin_lib import ast import os from situations.selfcontain import situation1 from situations.std_pub_level import situation2 from situations.classlevel import situation3 from situations.filelevel import situation4, situation4NotClass def get_import(astFile): list_tot = [] for node in ast.walk(astFile): if isinstance(node, ast.Import): imports = node.names for tempImport in imports: name = tempImport.name as_name = tempImport.asname if as_name is not None: list_tot.append(str(as_name)) else: list_tot.append(str(name)) elif isinstance(node, ast.ImportFrom): from_module = node.module if from_module is None: from_module = "." imports = node.names for tempImport in imports: name = tempImport.name as_name = tempImport.asname if as_name is not None: list_tot.append(from_module + "." + str(as_name)) else: list_tot.append(from_module + "." + str(name)) return list_tot def parse_class(classAst): list_var = [] list_func_def = [] for node in ast.walk(classAst): if isinstance(node, ast.FunctionDef): list_func_def.append(str(node.name)) for node in ast.walk(classAst): if isinstance(node, ast.Assign): for name in node.targets: if isinstance(name, ast.Name): # list_var.append() pass if isinstance(name, ast.Attribute): if hasattr(name, "value"): if isinstance(name.value, ast.Name): if name.value.id == "self": list_var.append(name.attr) return list_var, list_func_def def parse_class_in_file(file_ast): list_class = [] for node in ast.walk(file_ast): if isinstance(node, ast.ClassDef): list_class.append(str(node.name)) return list_class def parse_var_file(file_ast): list_return = [] for node in file_ast.body: if isinstance(node, ast.Assign): for name in node.targets: if isinstance(name, ast.Name): list_return.append(name.id) return list_return def parse_method_file(file_ast): list_return = [] for node in file_ast.body: if isinstance(node, ast.FunctionDef): list_return.append(node.name) return list_return def parse_file_with_method(func_lineno, func_code, file_content, list_std, list_third, list_builtin): # return_dict = {"self_contained": False, "standard_lib": False, "public_lib": False, "class": False, "file": False, # "project_level": False, # "other": False} try: ast_file = ast.parse(file_content) except: return list_import = get_import(ast_file) list_white=[] list_white.extend(list_std) list_white.extend(list_third) list_class = parse_class_in_file(ast_file) list_method_file=parse_method_file(ast_file) for node in ast.walk(ast_file): if isinstance(node, ast.ClassDef): list_var_class, list_method_class = parse_class(node) # print(list_method_class) list_class.remove(str(node.name)) for nodeMethod in ast.walk(node): if isinstance(nodeMethod, ast.FunctionDef): return_dict = {"self_contained": False, "standard_lib": False, "public_lib": False, "class": False, "file": False} line_no = nodeMethod.lineno if line_no != func_lineno: continue #print("start:" + node.name + "." + nodeMethod.name) message1, isT1 = situation1(nodeMethod, list_builtin) message2_1, isT2_1 = situation2(nodeMethod, list_import, list_std, list_builtin) message2_2, isT2_2 = situation2(nodeMethod, list_import, list_third, list_builtin) message3, isT3 = situation3(nodeMethod, list_import, list_white, list_var_class, list_method_class, list_builtin, list_class) message4, isT4 = situation4(nodeMethod, list_import, list_white, list_var_class, list_method_class, list_import, list_builtin, list_class, list_method_file) # print(message1) # print(message2_1) # print(message2_2) # print(message3) # print(message4) if isT1: return_dict['self_contained'] = True if isT1 and isT2_1: return_dict['standard_lib'] = True if isT1 and isT2_2: return_dict['public_lib'] = True if isT3: return_dict['class'] = True if isT4: return_dict['file'] = True # if not isT1 and not isT2_1 and not isT2_2 and not isT3 and not isT4: # return_dict['other'] = True #print(return_dict) return return_dict list_class.append(str(node.name)) for node in ast_file.body: if isinstance(node, ast.FunctionDef): return_dict = {"self_contained": False, "standard_lib": False, "public_lib": False, "class": False, "file": False} line_no = node.lineno if line_no != func_lineno: continue #print("start:" + node.name) message1, isT1 = situation1(node, list_builtin) message2_1, isT2_1 = situation2(node, list_import, list_std, list_builtin) message2_2, isT2_2 = situation2(node, list_import, list_std, list_builtin) message4, isT4 = situation4NotClass(node, list_import, list_white, list_import, list_builtin, list_method_file) if isT1: return_dict['self_contained'] = True if isT1 and isT2_1: return_dict['standard_lib'] = True if isT1 and isT2_2: return_dict['public_lib'] = True if isT4: return_dict['file'] = True # if not isT1 and not isT2_1 and not isT2_2 and not isT4: # return_dict['other'] = True # print(message1) # print(message2_1) # print(message2_2) # print(message4) # print(return_dict) return return_dict def input(data): func_name = data['name'] func_lineno = data['lineno'] func_code = data['code'] file_content = data['file_content'] return func_name, func_lineno, func_code, file_content def parse_single_method(data): file_path = "./data/3.6.1-static.json" third_path = "./data/pypi_downloads_last_180_days_20211117.csv" buildin_path = "./data/builtin.txt" list_third = get_third_lib(third_path) list_std = get_std_lib(file_path) list_builtin = get_buildin_lib(buildin_path) func_name, func_lineno, func_code, file_content = input(data) return parse_file_with_method(func_lineno, func_code, file_content, list_std, list_third, list_builtin) if __name__ == "__main__": data = {} ## input!!! # f = open("D:\python-master\python-master\kubernetes\\base\dynamic\\discovery.py", 'r') f = open("D:\AbsBigCloneBench-main\AbsBigCloneBench-main\\getASTsentenceForPOJ.py", 'r') file_content = f.read() f.close() data['name']="" data['lineno']=42 data['code']="" data['file_content']=file_content return_dict = parse_single_method(data) print(return_dict)