featureselection
发布时间:2020-12-20 11:02:15 所属栏目:Python 来源:网络整理
导读:# !/usr/bin/env python # -*- coding: utf-8 -*- ## Copyright 2019 The FATE Authors. All Rights Reserved. ## Licensed under the Apache License,Version 2.0 (the "License"); # you may not use this file except in compliance with the License. #
#!/usr/bin/env python # -*- coding: utf-8 -*- # # Copyright 2019 The FATE Authors. All Rights Reserved. # # Licensed under the Apache License,Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing,software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import subprocess import sys import time # from examples.load_file.load_file import load_file home_dir = os.path.split(os.path.realpath(__file__))[0] config_path = home_dir + ‘/conf‘ data_path = home_dir + ‘/../data‘ load_file_program = home_dir + ‘/../load_file/load_file.py‘ # data_set = ‘breast‘ data_set = ‘default_credit‘ # data_set = ‘give_credit‘ mode = ‘fit‘ # mode = ‘transform‘ def make_config_file(work_mode,job_id,role,guest_partyid,host_partyid): with open(config_path + ‘/{}_runtime_conf.json‘.format(role),‘r‘,encoding=‘utf-8‘) as load_f: role_config = json.load(load_f) if role == ‘guest‘: role_config[‘local‘][‘party_id‘] = guest_partyid data_suffix = ‘b‘ else: role_config[‘local‘][‘party_id‘] = host_partyid data_suffix = ‘a‘ role_config[‘role‘][‘host‘][0] = host_partyid role_config[‘role‘][‘guest‘][0] = guest_partyid role_config[‘WorkFlowParam‘][‘work_mode‘] = int(work_mode) # role_config[‘FeatureSelectionParam‘][‘method‘] = mode role_config[‘WorkFlowParam‘][‘train_input_table‘] = "{}_{}_{}".format(data_set,job_id) # guest_config_path = config_path + ‘/guest_runtime_conf.json_‘ + str(job_id) role_config_path = "{}/{}_runtime_conf.json_{}".format( config_path,job_id ) with open(role_config_path,‘w‘,encoding=‘utf-8‘) as json_file: json.dump(role_config,json_file,ensure_ascii=False) with open(config_path + ‘/load_file.json‘,encoding=‘utf-8‘) as load_f: load_config = json.load(load_f) load_config[‘work_mode‘] = work_mode # load_config[‘file‘] = data_path + ‘/‘ + data_set + ‘_b.csv‘ load_config[‘file‘] = "{}/{}_{}.csv".format(data_path,data_set,data_suffix) load_config[‘table_name‘] = "{}_{}_{}".format(data_set,job_id) # load_file_role = config_path + ‘/load_file.json_guest_‘ + str(job_id) load_file_path = "{}/load_file.json_{}_{}".format(config_path,job_id) with open(load_file_path,encoding=‘utf-8‘) as json_file: json.dump(load_config,ensure_ascii=False) return role_config_path,load_file_path def attach_all_result_log(running_process,role): while True: poll = running_process.poll() if poll is None: print(‘Workflow is still running‘) time.sleep(5) else: break log_dir = home_dir + ‘/../../logs/{}‘.format(job_id) role = role.lower() if role == ‘guest‘: role_list = [‘Guest‘,‘guest‘,‘GUEST‘] else: role_list = [‘Host‘,‘host‘,‘HOST‘] result_log_list = [] for log_file in os.listdir(log_dir): # print(‘current log_file: {}‘.format(log_file)) with open(log_dir + ‘/‘ + log_file,encoding=‘utf-8‘) as f: for log_line in f: if ‘Result‘ in log_line: for r in role_list: if r in log_line: result_log_list.append(log_line) break log_list = sorted(result_log_list,key=get_time) for log_line in log_list: print(log_line) def get_time(log_line): time_str = log_line.split(‘ - ‘)[0] time_str = time_str[1:] time_str.strip() time_sec,time_mili_sec = time_str.split(‘,‘) timestamp = time.mktime(time.strptime(time_sec,‘%Y-%m-%d %H:%M:%S‘)) timestamp += float(time_mili_sec) / 1000 return timestamp def load_file(load_file_path): load_process = subprocess.Popen(["python",load_file_program,"-c",load_file_path,]) # load_process.communicate() returncode = load_process.wait() print("Load file return code : {}".format(returncode)) if __name__ == ‘__main__‘: work_mode = int(sys.argv[1]) jobid = sys.argv[2] role = sys.argv[3] guest_partyid = int(sys.argv[4]) host_partyid = int(sys.argv[5]) role_config_path,load_file_path = make_config_file(work_mode,jobid,host_partyid) load_file(load_file_path) work_path = home_dir + ‘/../../workflow/hetero_feature_selection_workflow/‘ ‘hetero_feature_selection_{}_workflow.py‘.format(role) p = subprocess.Popen(["python",work_path,role_config_path,"-j",jobid ]) attach_all_result_log(p,role) (编辑:李大同) 【声明】本站内容均来自网络,其相关言论仅代表作者个人观点,不代表本站立场。若无意侵犯到您的权利,请及时与联系站长删除相关内容! |