#!/usr/bin/env python # ！-*-coding:utf-8 -*- # !@Date : 2018/11/9 0009 下午 17:08 # !@Author : Damon.guo # !@File : updateStandard.py import ConfigParser from optparse import OptionParser from subprocess import PIPE, Popen import time import os import sys import xml.dom.minidom import signal import codecs import shutil import zipfile import time import json reload(sys) sys.setdefaultencoding('utf-8') # 配置文件模板 server_conf_str = """ # 配置通过 # 可以注释不生效 [conf] # node服务和node服务的配置文件 node_conf = /python-project/node.conf # 备份上一次的应用目录 bak_dir = /app/bak/ # 备份文件控制版本数 bak_num = 5 # 服务检查次数 check_time = 3 # 日志路径 logs_path = /data/logs/ #ansible 主机文件 ansibile_host = /etc/ansible/hosts # maven 执行命令路径 mvn = /app/apache-maven-3.5.0/bin/mvn # java 执行命令路径 java = /app/jdk1.8.0_121/bin/java # nohup 命令路径 nohup = /usr/bin/nohup # node执行命令路径 node = /opt/node-v9.5.0-linux-x64/bin/node npm = /opt/node-v9.5.0-linux-x64/bin/npm # pthon 执行命令路径 python = /usr/bin/python # 定义远程执行脚本路径 ， remote_py = /python-project/nodeService.py """ node_conf_str = """ [bargainssr] builddir = /kilimall/procjet/node/bargainssr/ deploydir = /kilimall/procjet/node/bargainssr/ giturl = http://git.kilimall.com/kilimall/bargainssr.git devnodename = node1 testnodename = node2 pronodename = node3 """ def getOptions(): parser = OptionParser() parser.add_option("-n", "--serverName", action="store", dest="serverName", default=False, help="serverName to do") parser.add_option("-a", "--action", action="store", dest="action", default=False, help="action -a [checkout,pull,push,master,install]") # # parser.add_option("-v", "--versionId", action="store", # dest="versionId", # default=False, # help="-v versionId") parser.add_option("-b", "--branchName", action="store", dest="branchName", default=False, help="-b branchName") # jar 服务启动区分环境 读取的配置不一样 parser.add_option("-e", "--envName", action="store", dest="envName", default=False, help="-e envName") options, args = parser.parse_args() return options, args def execSh(cmd): # 执行SH命令 try: print "执行ssh 命令 %s" % cmd p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) except Exception, e: print e, sys.exit() return p.communicate() def checkMaster(): # 获取项目分支是否为master cmd = "git branch" stdout, stderr = execSh(cmd) print "out:", stdout branch_list = [i.strip() for i in stdout.split("\n") if i] if "* master" in branch_list: print "已经在master 分支" return True print "err", stderr return False # 初始化项目主应用可用于php部署， def initProject(serverName): # 新机器 或者新目录项目部署 print "master install:%s" % serverName if os.path.exists(nodeConf): projectDict = readConf(nodeConf) print projectDict else: print "%s is not exists" % nodeConf print node_conf_str sys.exit() deployDir = projectDict[serverName]["builddir"] gitUrl = projectDict[serverName]["giturl"] if not os.path.exists(deployDir): os.mkdir(deployDir) os.chdir(deployDir) print "部署路径：", os.getcwd() stdout, stderr = execSh("git status .") if stdout: print"out：\n%s" % stdout print "当前目录：%s,已经存在git仓库请检查!" % deployDir return True if stderr: print "没有git仓库，下一步" print"out：%s" % stderr print "初始化本地仓库" ReturnExec("git init") print"本地git仓库当前项目认证" config_cmd = "git config --local credential.helper store" ReturnExec(config_cmd) print "拉取代码" pull_cmd = "git pull %s" % gitUrl ReturnExec(pull_cmd) print "添加远程仓库地址" add_remote_cmd = "git remote add origin %s" % gitUrl ReturnExec(add_remote_cmd) print "获取分支" fetch_cmd = "git fetch" ReturnExec(fetch_cmd) print "关联本地master分支与远程master" upstream_cmd = "git branch --set-upstream-to=origin/master master" ReturnExec(upstream_cmd) print "获取 最新master分支" pull_m_cmd = "git pull" ReturnExec(pull_m_cmd) def ReturnExec(cmd): stdout, stderr = execSh(cmd) if stdout: print 80 * "#" print "out:%s " % stdout if stderr: print 80 * "#" print "err:%s" % stderr def readStdin(): input_str = raw_input("确认执行操作：Y/N") return input_str.strip().lower() # 合并分支至master def mergeBranch(serverName, branchName): deployDir = projectDict[serverName]["deploydir"] fetch_cmd = "git fetch origin %s" % branchName checkout_b_cmd = "git checkout %s" % branchName pull_cmd = "git pull" checkout_m_cmd = "git checkout master" merge_cmd = "git merge origin/%s" % branchName push_cmd = "git push origin master" try: print "切换工作目录" print deployDir os.chdir(deployDir) # 切换工做目录 print os.getcwd() except Exception, e: print e sys.exit() print "取分支" stdout, stderr = execSh(fetch_cmd) print stdout if "fatal" in stderr: print stderr print "检查分支 branchname:%s" % branchName sys.exit() # ReturnExec(fetch_cmd) # 更新分支 print "更新本地 分支" ReturnExec(pull_cmd) # 切换至master分支 if not checkMaster(): print "切换至master分支" ReturnExec(checkout_m_cmd) # 更新master分支 print "更新master分支" ReturnExec(pull_cmd) # 合并分支至master print "是否合并分支至master" ReturnExec(merge_cmd) # 提交合并的master 至源端git库 # 需要加确认 文件修改，在判断是否推送源端 print "是否提交合并的master 至源端git库" option = readStdin() if option != "y": sys.exit() ReturnExec(push_cmd) # 更新应用节点代码 # print "是否更新应用节点代码" # option = readStdin() # # if option != "y": # sys.exit() # updatecmd = 'ansible %s -m shell -a "cd %s;sudo git pull";' % (deployNodeName, deployDir) # ReturnExec(updatecmd) # 检查配置文件 def confCheck(cf, section, option): if not cf.options(section): print "no section: %s in conf file" % section sys.exit() try: options = cf.get(section, option) except ConfigParser.NoOptionError: print "no option in conf %s " % option sys.exit() if not options: print "options:(%s) is null in section:(%s)" % (option, section) return False else: return True # 读取配置文件 def readConf(serverConf): if not fileExists(serverConf): sys.exit() cf = ConfigParser.ConfigParser() try: cf.read(serverConf) except ConfigParser.ParsingError, e: print e print "请检查服务配置文件： %s" % serverConf sys.exit() serverNameDict = {} optinsDict = {} for serverName in cf.sections(): # print 'serverName:%s' % serverName for optins in cf.options(serverName): # 取服务名下的对应的配置和参数 if not confCheck(cf, serverName, optins): sys.exit() value = cf.get(serverName, optins) optinsDict[optins] = value serverNameDict[serverName] = optinsDict optinsDict = {} return serverNameDict # 本地安装项目初始化 def installServerName(serverName): serverNameDict = projectDict[serverName] deployDir = serverNameDict["deploydir"] # print projectDict if os.path.exists(deployDir): if dir_is_null(deployDir): print "%s 安装成功" % serverName return True else: print "%s 已经安装，请检查!" % serverName return False else: # os.mkdirs(deployDir) os.makedirs(deployDir) print "%s 安装成功" % serverName return True def getPid(serverName): deployDir = projectDict[serverName]["deploydir"] cmd = "pgrep -f %snode_modules/.bin/" % deployDir # cmd = "pgrep -f %s/war/" % deployDir pid, stderr = execSh(cmd) if pid: # string(pid,) print "取得 PID:%s" % pid return int(pid) def stopServer(serverName): # 停止服务 先正常停止，多次检查后 强制杀死！ pid = getPid(serverName) if pid: print "服务:%s,即将停止 pid:%s" % (serverName, pid) try: cmd = "sudo kill -9 %s" % pid killstdout, killsterr = execSh(cmd) if killstdout: print killstdout if killsterr: print killsterr # os.kill(pid, signal.SIGKILL) print '停止服务:%s, pid:%s' % (serverName, pid) except OSError, e: print 'No such as server!', e for i in xrange(1, checkTime + 1): print "检查服务是否停止， num:%s" % i if not getPid(serverName): print "服务:%s,停止成功" % serverName return True if getPid(serverName): print "服务:%s 停止失败，请检查服务" % serverName return False else: print "服务:%s,成功停止" % serverName return True def startServer(serverName): serverNameDict = projectDict[serverName] deploydir = serverNameDict["deploydir"] if not os.path.exists(deploydir): print "%s is not exits" % deploydir # sys.exit() return False serverlogpath = os.path.join(logsPath, serverName) if getPid(serverName): print "%s 已经启动,请先停止！ " % serverName return False else: os.chdir(deploydir) print "workdir：%s" % os.getcwd() print "启动服务：%s" % serverName cmd = "%s %s run start >%s.out 2>&1 &" % (nohup,npm, serverlogpath) stdout, stderr = execSh(cmd) if stdout: print "stdout:%s" % stdout if stderr: print "ster:%s" % stderr for i in xrange(1, checkTime + 1): print "循环检查服务启动状态：%s 次" % i time.sleep(checkTime+30) getPid(serverName) if getPid(serverName): # 需要目标服务器 在env 环境找到node执行命令 否则会报错。无法执行远程启动 print "启动服务：%s 成功" % serverName return True else: print "目标服务器尝试执行 'ln -s %s /usr/bin/node' 在重试" % node print "目标服务器尝试执行 'ln -s %s /usr/bin/npm' 在重试" % npm print "启动服务： %s 失败" % serverName return False def versionSort(list): # 对版本号排序 控制版本的数量 from distutils.version import LooseVersion vs = [LooseVersion(i) for i in list] vs.sort() return [i.vstring for i in vs] def getVersion(serverName): # bakdeployRoot = getDeploymentTomcatPath(serverName)["bakServerDir"] bakdeployRoot = os.path.join(bakDir, serverName) versionIdList = [] try: for i in os.listdir(bakdeployRoot): if i.split(".")[0] == serverName: versionId = i.split(".")[1] versionIdList.append(versionId) except: return [] if not versionIdList: return [] return versionSort(versionIdList) # 返回版本号升序列表 def getBackVersionId(serverName): date = time.strftime("%Y-%m-%d") versionIdList = getVersion(serverName) if not versionIdList: return 1 else: # 同一日期下的最新版本+1 if date != versionSort(versionIdList)[-1].split("-V")[0]: return 1 else: return int(versionIdList[-1].split("-")[-1].split("V")[-1]) + int(1) def TimeStampToTime(timestamp): # 时间戳转换为时间 timeStruct = time.localtime(timestamp) return time.strftime('%Y-%m-%d %H:%M:%S', timeStruct) # 返回时间戳 def getTimeStamp(filePath): # import unicode filePath = unicode(filePath, 'utf8') t = os.path.getmtime(filePath) # return t return TimeStampToTime(t) # 清理历史多余的备份文件 def cleanHistoryBak(serverName): # bakServerDir = getDeploymentTomcatPath(serverName)["bakServerDir"] bakServerDir = os.path.join(bakDir, serverName) VersinIdList = getVersion(serverName) # print VersinIdList if VersinIdList: if len(VersinIdList) > int(bakNum): cleanVersionList = VersinIdList[0:abs(len(VersinIdList) - int(bakNum))] for i in cleanVersionList: bakWarPath = os.path.join(bakServerDir, "%s.%s") % (serverName, i) if os.path.exists(bakWarPath): print "clean history back WAR: %s" % bakWarPath # os.remove(bakWarPath) if os.path.isdir(bakWarPath): shutil.rmtree(bakWarPath) elif os.path.isfile(bakWarPath): os.remove(bakWarPath) else: print "othe err" else: pass else: print "%s is not bak War" % serverName def copyFile(sourfile, disfile): try: print "copy file:%s,to:%s" % (sourfile, disfile) shutil.copy2(sourfile, disfile) # % ( disfile, time.strftime("%Y-%m-%d-%H%M%S")) except Exception, e: print e, sys.exit(1) def copyDir(sourDir, disDir): try: print "copy Dir:%s,to:%s" % (sourDir, disDir) # shutil.copy2(sourDir, disDir) # % ( disfile, time.strftime("%Y-%m-%d-%H%M%S")) # shutil.copytree(sourDir, disDir) cmd = "cp -ar %s %s " %(sourDir, disDir) ReturnExec(cmd) except Exception, e: print e, sys.exit(1) def checkServer(serverName): deployDir = projectDict[serverName]["deploydir"] if os.path.exists(deployDir): return True else: return False # 判断目录是否为空 def dir_is_null(path): # print os.listdir(path) if os.path.exists(path): if os.listdir(path): # 不为空 False return False # 是空返回True return True def backWar(serverName): # 部署的包 # init(serverconf) # deployDir = serverNameDict[serverName]["deploydir"] # print projectDict try: deployWar = projectDict[serverName]["deployFile"] # deployWar = os.path.join(deployDir, deployFile) except: deployWar = projectDict[serverName]["deploydir"] # 备份包路径 bakServerDir = os.path.join(bakDir, serverName) versionId = getBackVersionId(serverName) # 同一日期下的最新版本 try: lastVersinId = getVersion(serverName)[-1] except: # 获取 备份文件列表 如果没有备份 返回备份起始版本1 lastVersinId = [time.strftime("%Y-%m-%d-") + "V1"][-1] bakdeployRootWar = os.path.join(bakServerDir, "%s.%sV%s") % (serverName, time.strftime("%Y-%m-%d-"), versionId) lastbakdeployRootWar = os.path.join(bakServerDir, "%s.%s") % (serverName, lastVersinId) if not checkServer(serverName): print "%s is not install" % serverName else: if os.path.exists(deployWar) and not dir_is_null(deployWar): if not os.path.exists(lastbakdeployRootWar): print "back %s >>> %s" % (deployWar, bakdeployRootWar) # copyFile(deployWar, bakdeployRootWar) if os.path.isdir(deployWar): copyDir(deployWar, bakdeployRootWar) elif os.path.isfile(deployWar): copyFile(deployWar, bakdeployRootWar) else: # 判断 最后一次备份和现在的文件是否 修改不一致，如果一致就不备份， if not getTimeStamp(deployWar) == getTimeStamp(lastbakdeployRootWar): print "back %s >>> %s" % (deployWar, bakdeployRootWar) if os.path.isdir(deployWar): copyDir(deployWar, bakdeployRootWar) elif os.path.isfile(deployWar): copyFile(deployWar, bakdeployRootWar) cleanHistoryBak(serverName) if os.path.exists(bakdeployRootWar): print "back %s sucess" % bakdeployRootWar else: print "back %s fail" % deployWar else: # print getVersion(serverName) print "File:%s is not modify,not need back" % deployWar else: print "path %s is null or %s is not exists" % (deployWar, bakdeployRootWar) def rollBack(serverName, versionId=""): # 为方便调用需要出示化服务字典，目前暂无优化，待优化，结合部署脚本使用 # init(serverconf) # dirDict = getDeploymentTomcatPath(serverName) dirDict = projectDict[serverName] versionList = getVersion(serverName) # print versionList if not versionList: print "Not Back war File :%s" % serverName else: if not versionId: versionId = versionList[-1] # bakdeployWar = os.path.join(dirDict["bakServerDir"],"%s.%s") % (serverName,versionId) bakdeployWar = os.path.join(bakDir, serverName, "%s.%s") % (serverName, versionId) # os.path.join(bakDir,serverName) # deployRootWar = dirDict["deploydir"] try: deployRootWar = dirDict["deployFile"] # deployRootWar = os.path.join(deployRootWar, deployFile) except: deployRootWar = dirDict["deploydir"] if not os.path.exists(bakdeployWar): print "File:%s is not exits" % bakdeployWar sys.exit() if os.path.exists(deployRootWar): # os.removedirs(deployRootWar) print "clean history file: %s " % deployRootWar if os.path.isdir(bakdeployWar): if os.path.exists(deployRootWar): shutil.rmtree(deployRootWar) copyDir(bakdeployWar, deployRootWar) elif os.path.isfile(bakdeployWar): os.remove(deployRootWar) copyFile(bakdeployWar, deployRootWar) # chown_cmd = "sudo chown ec2-user.ec2-user -R %s" % deployRootWar # stdout, stderr = execSh(chown_cmd) # if stdout: # print "stdout：%s" % stdout # if stderr: # print "stederr： %s" % stderr if os.path.exists(deployRootWar): print "RollBack Sucess,update serverName:%s" % serverName print "Rollback Version:%s " % versionId else: print "check File:%s ,rollback Faile" % deployRootWar # 判断目录是否为空 def dir_is_null(path): # print os.listdir(path) if os.listdir(path): # 不为空 False return False # 是空返回True return True def main(serverName, branchName, action): if action == "init": # 主服务项目部署 用代码分支合并，mvn 构建，在主服务器上 initProject(serverName) elif action == "merge": # 主服务项目合并分支至master mergeBranch(serverName, branchName) elif action == "install": # 用于远端机器部署项目 installServerName(serverName) elif action == "restart": stopServer(serverName) startServer(serverName) elif action == "start": startServer(serverName) elif action == "stop": stopServer(serverName) elif action == "back": stopServer(serverName) backWar(serverName) elif action == "getback": versionlist = getVersion(serverName) if not versionlist: print "%s not back" % serverName else: print "%s has back version:%s" % (serverName, versionlist) elif action == "rollback": #stopServer(serverName) rollBack(serverName) # startServer(serverName) else: print "action just [install,init,back,rollback，getback，start,stop,restart]" sys.exit() # 输出服务配置文件中的服务名 def printServerName(projectDict): serverNameList = [] # print projectDict for serverName, serverNameDict in projectDict.items(): print "可执行服务名：%s" % serverName serverNameList.append(serverName) # 返回服务名列表，可以在后期处理进行排序，考虑服务启动的顺序 return serverNameList # 检查文件是否存在 def fileExists(filePath): if not os.path.exists(filePath): print "文件：%s 不存在，请检查" % filePath return False return True def init(serverconf): if os.path.exists(serverconf): confDict = readConf(serverconf) return confDict else: print "%s is not exists" % serverconf print server_conf_str sys.exit() if __name__ == "__main__": serverconf = "/data/init/nodeServer.conf" # print os.getcwd() # sys.exit() confDict = init(serverconf)["conf"] options, args = getOptions() action = options.action # version = options.versionId serverName = options.serverName branchName = options.branchName envName = options.envName try: nodeConf = confDict["node_conf"] global bakDir, bakNum, checkTime, logsPath, mvn, java, nohup, npm, node bakDir = confDict["bak_dir"] bakNum = int(confDict["bak_num"]) checkTime = int(confDict["check_time"]) logsPath = confDict["logs_path"] mvn = confDict["mvn"] nohup = confDict["nohup"] npm = confDict["npm"] node = confDict["node"] # node = confDict["node"] except Exception, e: print "%s 错误" % e sys.exit() if not os.path.exists(logsPath): os.makedirs(logsPath) if not os.path.exists(bakDir): os.makedirs(bakDir) if os.path.exists(nodeConf): projectDict = readConf(nodeConf) # print projectDict else: print "%s is not exists" % nodeConf print node_conf_str sys.exit() if not action: print "参数执行操作 -a action [install,init,back,rollback，getback，start,stop,restart]" sys.exit() elif not serverName: print "参数服务名 -n servername " printServerName(projectDict) sys.exit() else: # if action == "start" or action == "restart" or action == "rollback": # if not envName: # print "参数执行操作 -e envName [dev,test,pro]" # sys.exit() if serverName == "all": # 进行升序排列 serverlist = sorted(projectDict.keys()) for serName in serverlist: main(serName, branchName, action) else: if not projectDict.has_key(serverName): print "没有服务名：%s" % serverName printServerName(projectDict) sys.exit() main(serverName, branchName, action)