diff --git a/.gitea/workflows/build_client.yml b/.gitea/workflows/build_client.yml index 0c79d8c..610e49b 100644 --- a/.gitea/workflows/build_client.yml +++ b/.gitea/workflows/build_client.yml @@ -72,20 +72,26 @@ jobs: run: | apt-get update apt-get install jq - - name: Download settings + - name: Download env run: | + pwd curl -H 'accept: application/json' -H 'authorization: Basic cGFraW46YWRtaW4xMjM=' "$(curl -X 'GET' \ - 'https://pakin-inspiron-15-3530.tail360bd.ts.net/api/v1/repos/pakin/taobin_recipe_manager/releases/19/assets/73' \ + ${{ secrets.DEPLOY_KEYS_DL }} \ -H 'accept: application/json' \ - -H 'authorization: Basic cGFraW46YWRtaW4xMjM=' | jq -r '.browser_download_url')" | jq -r '.body.env' | base64 -d > ./server/app.env - curl -H 'accept: application/json' -H 'authorization: Basic cGFraW46YWRtaW4xMjM=' "$(curl -X 'GET' \ - 'https://pakin-inspiron-15-3530.tail360bd.ts.net/api/v1/repos/pakin/taobin_recipe_manager/releases/19/assets/73' \ - -H 'accept: application/json' \ - -H 'authorization: Basic cGFraW46YWRtaW4xMjM=' | jq -r '.browser_download_url')" | jq -r '.body.secret' | base64 -d > ./server/client_secret.json - - name: Run Golang tests + -H 'authorization: Basic cGFraW46YWRtaW4xMjM=' | jq -r '.browser_download_url' | sed -e 's/http/https/g' | sed -e 's/100.64.210.13\:3001/pakin-inspiron-15-3530.tail360bd.ts.net/g')" | jq -r '.body.env' | base64 -d > ./server/app.env + cat ./server/app.env + - name: Download secret run: | - cd server - go test -v ./... + pwd + curl -H 'accept: application/json' -H 'authorization: Basic cGFraW46YWRtaW4xMjM=' "$(curl -X 'GET' \ + ${{ secrets.DEPLOY_KEYS_DL }} \ + -H 'accept: application/json' \ + -H 'authorization: Basic cGFraW46YWRtaW4xMjM=' | jq -r '.browser_download_url' | sed -e 's/http/https/g' | sed -e 's/100.64.210.13\:3001/pakin-inspiron-15-3530.tail360bd.ts.net/g')" | jq -r '.body.secret' | base64 -d > ./server/client_secret.json + cat ./server/client_secret.json + # - name: Run Golang tests + # run: | + # cd server + # go test -v ./... - name: Build and push run: | pwd diff --git a/client/src/app/features/merge/merge.component.html b/client/src/app/features/merge/merge.component.html index f510ed8..eb5f08f 100644 --- a/client/src/app/features/merge/merge.component.html +++ b/client/src/app/features/merge/merge.component.html @@ -15,10 +15,10 @@ > [{{ getCommitAttr(commit, "Created_at") }}] [{{ getCommitAttr(commit, "Editor") - }}] | {{ getCommitAttr(commit, "Msg") }} + }}] | {{ getCommitAttr(commit, "Msg") }} | {{ getCommitAttr(commit, "Id") }} - +
@@ -29,13 +29,13 @@ > @@ -109,6 +109,8 @@
{{ selectedCommit }} + {{ anotherSelectedSource}} + {{ anotherSelectedSource.startsWith('coffeethai02_') }}
@@ -123,7 +125,7 @@ (recipeListFormChange)="onRecipeListFormChange($event)" >
-
+
-
- + + + + +
+ + -
+
diff --git a/client/src/app/features/merge/merge.component.ts b/client/src/app/features/merge/merge.component.ts index 9f5563b..0359b41 100644 --- a/client/src/app/features/merge/merge.component.ts +++ b/client/src/app/features/merge/merge.component.ts @@ -83,6 +83,8 @@ export class MergeComponent // from another source anotherTargetRecipe: any = {}; + reloadMainSource = true; + // -------------------- current selection // currentTargetOfMaster: any = undefined; @@ -169,6 +171,14 @@ export class MergeComponent }); } + // reload data of main-source + triggerReload() { + this.reloadMainSource = false; + setTimeout(() => { + this.reloadMainSource = true; + }, 0); + } + async ngOnInit(): Promise { // fetch related product codes } @@ -200,6 +210,15 @@ export class MergeComponent // get patch map keys getPatchMapKeys = () => Object.keys(this.patchMap); + // get patch map keys, if have the same productCode + getPatchMapKeysIfProductCode() { + const keys = Object.keys(this.patchMap).filter(selectedCommit => { + return this.patchMap[selectedCommit].productCode === this.selectedProductCode; + }); + // console.log("Filtered PacthMap : ", keys); + return keys; + } + // check if load patch keys isPatchMapKeysLoaded = () => this.getPatchMapKeys().length > 0; testLoadCheck = () => @@ -260,19 +279,26 @@ export class MergeComponent selectCommit = (commit: any) => { // console.log('select commit', commit.target.value); this.selectedCommit = commit.target.value; + this.selectedProductCode = this.getCommitAttr(this.selectedCommit,"contents").productCode + // reload data of main-source + this.triggerReload(); + console.log("selectedCommit target v : ",this.selectedCommit) + console.log("selectedProductCode : ",this.selectedProductCode) }; selectAnotherSource = (source: any) => { this.anotherSelectedSource = source.target.value; + console.log("another select target v : ",this.anotherSelectedSource) }; changeAnotherSource = async (source: any) => { - this.anotherSelectedSource = - 'coffeethai02_' + source.target.value + '.json'; - console.log( - 'another source: target version -> ', - this.anotherSelectedSource - ); + //base on change + this.anotherSelectedSource = 'coffeethai02_' + source.target.value + '.json'; + // const vset = '691' + // this.anotherSelectedSource = `coffeethai02_${source}.json`; + console.log('another source: target version -> ',this.anotherSelectedSource); + console.log("anotherTargetRecipe : ",this.anotherTargetRecipe) + // activate fetch for (let pd of this.getProductCodesOfCommits()) { await this.getAnotherRecipeOfProductCode(pd); @@ -321,7 +347,8 @@ export class MergeComponent return { changeContext: undefined, skipZeroes: true, - toppingData: this.anotherTargetRecipe[pd!].ToppingSet, + // toppingData: this.anotherTargetRecipe[pd!].ToppingSet, + toppingData: this.getCommitAttr(this.selectedCommit, 'contents').ToppingSet, }; } } @@ -384,15 +411,17 @@ export class MergeComponent // test compare this.getPatchMapKeys().forEach((patchId) => { // compare with master - let cmp = compare( - this.anotherTargetRecipe[productCode!], - this.fullPatches[patchId].contents, - ['LastChange'] - ); - // save only what changes - this.changeMap[patchId + '_' + this.anotherSelectedSource] = { - changes: cmp, - }; + if (this.anotherTargetRecipe[productCode!]['productCode'] == this.fullPatches[patchId].contents['productCode']) { + let cmp = compare( + this.anotherTargetRecipe[productCode!], + this.fullPatches[patchId].contents, + ['LastChange'] + ); + // save only what changes + this.changeMap[patchId + '_' + this.anotherSelectedSource] = { + changes: cmp, + }; + } }); console.log('change map', this.changeMap); @@ -500,6 +529,7 @@ export class MergeComponent // selection empty? isSelectionEmpty(side: string): boolean { + console.log("isSelectionEmpty : ",side,this.selectionMap[this.selectedCommit + side]) return this.selectionMap[this.selectedCommit + side] == undefined || this.selectionMap[this.selectedCommit + side].length == 0; } diff --git a/client/src/app/features/recipes/recipe-details/recipe-list/recipe-list.component.html b/client/src/app/features/recipes/recipe-details/recipe-list/recipe-list.component.html index 211323b..0d5644b 100644 --- a/client/src/app/features/recipes/recipe-details/recipe-list/recipe-list.component.html +++ b/client/src/app/features/recipes/recipe-details/recipe-list/recipe-list.component.html @@ -61,7 +61,7 @@ >

Volume

@@ -77,7 +77,7 @@ >

Volume

@@ -122,7 +122,7 @@

Hot

@@ -139,7 +139,7 @@ >

Cold

@@ -161,7 +161,7 @@ >

{{ getTooltipForStirTime(getTypeForRecipeListAtIndex(i)) }}

@@ -301,7 +301,7 @@ > dict: + return {"accept":"application/json", "authorization": "Basic cGFraW46YWRtaW4xMjM="} + +def get_headers_for_blob() -> dict: + return {"authorization": "Basic cGFraW46YWRtaW4xMjM=", "Content-Type": "application/octet-stream"} + +def get_all_releases_url() -> str: + return f"{source}/api/v1/repos/pakin/taobin_recipe_manager/releases" + +def get_download_url(tag_id: int, asset_id: int) -> str: + return f"{source}/api/v1/repos/pakin/taobin_recipe_manager/releases/{tag_id}/assets/{asset_id}" + +# Flags +update_available_now = False + +# Logs +def save_to_log_file(msg: str): + with open("./patches/.updater.log", "a") as f: + f.write(f"{datetime.datetime.now(datetime.timezone(datetime.timedelta(hours=7)))} : {msg}\n") + +# Redis +redis_client = redis.StrictRedis(host="localhost", port=6379, db=0) +MAIN_CHANNEL = "updater.noti" +pubsub = redis_client.pubsub() +pubsub.subscribe(MAIN_CHANNEL) + +# Settings +settings = {} + +# ------------------------------------------------------------------------------------------------ + +def test_path_from_settings(): + # from settings + path_list = settings["path"].keys() + + for path in path_list: + logger.debug(f"Checking {path}") + if not os.path.exists(settings["path"][path]["path"]): + logger.warning(f"Deps.fallback: {settings['path'][path]["path"]}") + # retry with fallback + if "fallback_paths" in settings['path'][path].keys(): + for fallback_path in settings['path'][path]["fallback_paths"]: + + # check kind of fallback + path_kind = fallback_path.split(":")[0] + path_real = fallback_path.split(":")[1] + + if os.path.exists(path_real): + logger.info(f"Deps.fallback[{path_kind}]: {path_real} = ok") + break + else: + logger.warning(f"Deps.fallback: skipped {path}") + else: + # path ok( + logger.info(f"Deps: {path} = ok") + + +def find_latest_version(rels: list[dict]) -> str: + major = 0 + minor = 0 + patch = 0 + for rel in rels: + if rel["tag_name"].startswith("v"): + version = rel["tag_name"][1:].split(".") + if len(version) == 3: + if int(version[0]) > major: + major = int(version[0]) + minor = int(version[1]) + patch = int(version[2]) + elif int(version[0]) == major: + if int(version[1]) > minor: + minor = int(version[1]) + patch = int(version[2]) + elif int(version[1]) == minor: + if int(version[2]) > patch: + patch = int(version[2]) + + return f"{major}.{minor}.{patch}" + +def get_version(rels: list[dict], tag: str) -> dict: + # assert that tag is valid + is_in_correct_format = tag.startswith("v") and len(tag[1:].split(".")) == 3 + if is_in_correct_format: + for rel in rels: + if rel["tag_name"] == tag: + return rel + +# +def check_version_job() -> dict: + logger.info("Checking version") + result = requests.get(get_all_releases_url(), timeout=10, headers=get_headers()) + if result.status_code == 200: + # connect to server ok! + releases = result.json() + if len(releases) > 0: + # find the latest {major}.{minor}.{patch} + expected_version = find_latest_version(releases) + # get data from list + # version data + latest_version_data = get_version(releases, f"v{expected_version}") + + + latest_id = latest_version_data["id"] + latest_asset_id = latest_version_data["assets"][0]["id"] + + # get download url + download_url = get_download_url(latest_id, latest_asset_id) + # download + logger.info(f"Found version {expected_version} from source") + + dl_result = requests.get(download_url, timeout=10, headers=get_headers()) + if dl_result.status_code == 200: + # download ok! + # logger.debug(f"data: {dl_result.json()}") + + dl_content = dl_result.json() + + dl_link = dl_content["browser_download_url"] + # download link is not usable without tailscale vpn + # cut uri, start at /attachments + dl_link = f"{source}{dl_link[dl_link.find("/attachments"):]}" + # logger.info(f"Download link: {dl_link}") + + is_already_latest = False + old_version = "" + # check dl and latest + if os.path.exists("./patches/latest.version"): + with open("./patches/latest.version", "r") as f: + latest_version = f.read() + + if latest_version == None or latest_version == "": + logger.warning("No version found on running machine. Starting overwrite ...") + save_to_log_file("Warning: version.overwrite caused by missing version on latest.version.") + old_version = "" + else: + old_version = latest_version + + if latest_version == expected_version: + logger.info(f"Already at latest version {expected_version}"); + is_already_latest = True + else: + logger.info(f"Updating to {expected_version}") + + # write back to latest + with open("./patches/latest.version", "w") as f: + f.write(expected_version) + else: + logger.info(f"First time running {expected_version}") + with open("./patches/latest.version", "w") as f: + f.write(expected_version) + + # extract + if not is_already_latest: + + # download + downloaded = requests.get(dl_link, timeout=10, headers=get_headers_for_blob()) + + if downloaded.status_code == 200: + # download ok! + + # create patch folder + if os.path.exists("./patches") == False: + os.mkdir("./patches") + + with open(f"./patches/patch_cli_{expected_version}.zip", "wb") as f: + f.write(downloaded.content) + + # write down version + with open("./patches/downloaded.version", "w") as f: + f.write(expected_version) + + logger.debug("Downloaded") + else: + logger.debug(f"Failed to download {expected_version}") + + logger.info(f"Extracting {expected_version}") + + zip_file_path = f"./patches/patch_cli_{expected_version}.zip" + # extract zip + # mkdir client + if os.path.exists("./patches/client") == False: + os.mkdir("./patches/client") + + + with zipfile.ZipFile(zip_file_path, 'r') as zip_ref: + zip_ref.extractall('./patches/client') + + # ------------------------------------------------------------------------------------ + + + + # run + # logger.info(f"Running {expected_version}") + # this will check image tag and pull if diff + + logger.info("Pulling from registry") + registry_source = f"{source[(source.find('//') + 2):]}" + try: + + client = docker.from_env() + log_response = client.api.login(username=settings["secret"]["username"], password=settings["secret"]["password"], registry=source) + logger.debug(f"log_response: {log_response}") + # client.images.pull(f"{registry_source}/pakin/taobin_recipe_manager", tag=expected_version) + pull_prog = client.api.pull(f"{registry_source}/pakin/taobin_recipe_manager", tag=f"v{expected_version}") + logger.debug(f"Pulling: {pull_prog}") + except Exception as e: + logger.error(f"Failed to pull {expected_version}") + + with open("./patches/error.txt", "w") as f: + f.write(f"{datetime.datetime.now(datetime.timezone(datetime.timedelta(hours=7)))} : fail to update server\n") + # return {"status": "failed to pull"} + save_to_log_file("fail to update server") + + global update_available_now + update_available_now = True + + # clean up + logger.info(f"Cleaning up {expected_version}") + os.remove(f"./patches/patch_cli_{expected_version}.zip") + os.remove("./patches/downloaded.version") + + save_to_log_file(f"update success from {old_version} -> {expected_version}") + + + else: + logger.error(f"Failed to download {expected_version}") + return {"status": "failed at download"} + + if update_available_now: + # publish + redis_client.publish(MAIN_CHANNEL, "new_version") + + return {"status": "ok"} + + else: + logger.error("No release found") + return {"status": "no release found"} + # assert that tag is valid + else: + logger.error("Failed to connect to server") + return {"status": "failed to connect to server"} + +def running_jobs(): + global update_available_now + update_available_now = False + print("---------------------------------------------------------------------") + status = check_version_job() + logger.info(f"Status ({status['status']})") + print("---------------------------------------------------------------------") + if update_available_now: + redis_client.publish(MAIN_CHANNEL, "new_version") + + +def notification_job(): + msgs = pubsub.get_message(timeout=5) + if msgs != None and msgs['type'] == 'message': + msg = msgs["data"].decode("utf-8") + logger.info(f"{MAIN_CHANNEL}> {msg}") + else: + logger.info(f"{MAIN_CHANNEL}> idle") + + + + + +@asynccontextmanager +async def lifespan(app: FastAPI): + + # load settings + config = open("./updater.settings.json", "r") + + + loaded_settings = json.load(config) + settings['path'] = loaded_settings['path'] + + try: + secret = open("./updater.secrets", "r") + + secret_string = secret.read() + spl = secret_string.split(":") + + settings['secret'] = { + "username": spl[0], + "password": spl[1] + } + except: + logger.warning("No secrets found") + exit(1) + + test_path_from_settings() + + update_available_now = False + logger.info("Starting up. Check update first ...") + redis_client.publish(MAIN_CHANNEL, "updater.first_start") + status = check_version_job() + logger.info(f"Status ({status['status']})") + # + scheduler.add_job(notification_job, 'interval', seconds=10, id="notification") + scheduler.add_job(running_jobs, 'interval', minutes=10, id="check_version") + scheduler.start() + yield + # schedule.every().hours.do(check_version_job) + print("Shutting down") + +scheduler = BackgroundScheduler(jobs_default={'max_instances':2}) +app = FastAPI(title="Updater",lifespan=lifespan) + +logger= logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) +stream_handler = logging.StreamHandler(sys.stdout) +log_formatter = logging.Formatter("%(asctime)s [%(levelname)s] : %(message)s") +stream_handler.setFormatter(log_formatter) +logger.addHandler(stream_handler) + +@app.get("/") +async def main(): + + if update_available_now: + return "Update available" + + return "Version is up to date." + +@app.get("/job_status") +async def job_status(): + return f"{scheduler.get_job("check_version")}" + + +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=36528) \ No newline at end of file