fix: harden tat ops scripts

ops/deploy_via_tat.py

@@ -6,8 +6,10 @@ import base64
 import json
 import os
 import time
+from typing import Callable, TypeVar
 
 from tencentcloud.common import credential
+from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException
 from tencentcloud.common.profile.client_profile import ClientProfile
 from tencentcloud.common.profile.http_profile import HttpProfile
 from tencentcloud.tat.v20201028 import models as tat_models, tat_client
@@ -19,6 +21,7 @@ DEFAULT_REPO_URL = os.environ.get("MONITOR_REPO_URL", "git@gitea.haiyihy.com:hy/
 DEFAULT_REPO_REF = os.environ.get("MONITOR_REPO_REF", "main")
 DEFAULT_REMOTE_DIR = os.environ.get("MONITOR_REMOTE_DIR", "/opt/hy-app-monitor")
 DEFAULT_APP_PORT = os.environ.get("APP_PORT", "6666")
+T = TypeVar("T")
 
 
 def env_required(name: str) -> str:
@@ -37,6 +40,21 @@ def build_tat(secret_id: str, secret_key: str, region: str) -> tat_client.TatCli
     )
 
 
+def invoke(action: Callable[[], T], label: str, retries: int = 4, delay_seconds: int = 2) -> T:
+    last_error: Exception | None = None
+    for attempt in range(1, retries + 1):
+        try:
+            return action()
+        except TencentCloudSDKException as exc:
+            last_error = exc
+            if attempt == retries:
+                break
+            time.sleep(delay_seconds * attempt)
+    if last_error is not None:
+        raise last_error
+    raise RuntimeError(f"{label} failed without error")
+
+
 def remote_script(repo_url: str, repo_ref: str, remote_dir: str, app_port: str) -> str:
     return f"""set -euo pipefail
 
@@ -80,7 +98,7 @@ def run_tat_script(client: tat_client.TatClient, instance_id: str, script: str,
             }
         )
     )
-    resp = json.loads(client.RunCommand(req).to_json_string())
+    resp = json.loads(invoke(lambda: client.RunCommand(req), "RunCommand").to_json_string())
     invocation_id = resp["InvocationId"]
     deadline = time.time() + timeout_seconds + 120
 
@@ -96,7 +114,7 @@ def run_tat_script(client: tat_client.TatClient, instance_id: str, script: str,
                 }
             )
         )
-        data = json.loads(client.DescribeInvocationTasks(query).to_json_string())
+        data = json.loads(invoke(lambda: client.DescribeInvocationTasks(query), "DescribeInvocationTasks").to_json_string())
         tasks = data.get("InvocationTaskSet") or []
         if not tasks:
             continue

ops/open_port_via_tat.py

@@ -7,8 +7,10 @@ import json
 import os
 import time
 from pathlib import Path
+from typing import Callable, TypeVar
 
 from tencentcloud.common import credential
+from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException
 from tencentcloud.common.profile.client_profile import ClientProfile
 from tencentcloud.common.profile.http_profile import HttpProfile
 from tencentcloud.cvm.v20170312 import cvm_client, models as cvm_models
@@ -20,6 +22,7 @@ DEFAULT_INSTANCE_ID = os.environ.get("DEPLOY_INSTANCE_ID", "ins-ntmpcd3a")
 DEFAULT_REGION = os.environ.get("DEPLOY_REGION", "me-saudi-arabia")
 DEFAULT_PORT = int(os.environ.get("OPEN_PORT", "6666"))
 DEFAULT_CIDR = os.environ.get("OPEN_CIDR", "0.0.0.0/0")
+T = TypeVar("T")
 
 
 def env_required(name: str) -> str:
@@ -47,10 +50,25 @@ def build_tat(secret_id: str, secret_key: str, region: str) -> tat_client.TatCli
     )
 
 
+def invoke(action: Callable[[], T], label: str, retries: int = 4, delay_seconds: int = 2) -> T:
+    last_error: Exception | None = None
+    for attempt in range(1, retries + 1):
+        try:
+            return action()
+        except TencentCloudSDKException as exc:
+            last_error = exc
+            if attempt == retries:
+                break
+            time.sleep(delay_seconds * attempt)
+    if last_error is not None:
+        raise last_error
+    raise RuntimeError(f"{label} failed without error")
+
+
 def resolve_security_group_id(client: cvm_client.CvmClient, instance_id: str) -> str:
     req = cvm_models.DescribeInstancesRequest()
     req.from_json_string(json.dumps({"InstanceIds": [instance_id]}))
-    resp = json.loads(client.DescribeInstances(req).to_json_string())
+    resp = json.loads(invoke(lambda: client.DescribeInstances(req), "DescribeInstances").to_json_string())
     instances = resp.get("InstanceSet") or []
     if not instances:
         raise SystemExit(f"instance not found: {instance_id}")
@@ -105,7 +123,7 @@ client = vpc_client.VpcClient(
     ClientProfile(httpProfile=HttpProfile(endpoint='vpc.tencentcloudapi.com')),
 )
 
-req = models.AuthorizeSecurityGroupPoliciesRequest()
+req = models.CreateSecurityGroupPoliciesRequest()
 req.from_json_string(json.dumps({{
     'SecurityGroupId': payload['security_group_id'],
     'SecurityGroupPolicySet': {{
@@ -120,7 +138,7 @@ req.from_json_string(json.dumps({{
 }}))
 
 try:
-    client.AuthorizeSecurityGroupPolicies(req)
+    client.CreateSecurityGroupPolicies(req)
     print('security_group_rule=created')
 except TencentCloudSDKException as exc:
     if 'Duplicate' in str(exc) or 'already exists' in str(exc):
@@ -147,7 +165,7 @@ def run_tat_script(client: tat_client.TatClient, instance_id: str, script: str,
             }
         )
     )
-    resp = json.loads(client.RunCommand(req).to_json_string())
+    resp = json.loads(invoke(lambda: client.RunCommand(req), "RunCommand").to_json_string())
     invocation_id = resp["InvocationId"]
     deadline = time.time() + timeout_seconds + 120
 
@@ -163,7 +181,7 @@ def run_tat_script(client: tat_client.TatClient, instance_id: str, script: str,
                 }
             )
         )
-        data = json.loads(client.DescribeInvocationTasks(query).to_json_string())
+        data = json.loads(invoke(lambda: client.DescribeInvocationTasks(query), "DescribeInvocationTasks").to_json_string())
         tasks = data.get("InvocationTaskSet") or []
         if not tasks:
             continue