Create app.py

app.py

@@ -0,0 +1,224 @@
+import os
+
+os.system('pip install huggingface_hub')
+
+
+import shutil
+import time
+
+import gradio as gr
+from huggingface_hub import HfApi
+
+CACHE_DIR = "hf2ms_cache"
+LOCAL_DIR = "hf2ms_local"
+USERNAME = "dailingx"
+EMAIL = "468551414@qq.com"
+
+
+def clone_from_ms(
+        ms_token: str,
+        ms_repo_id: str,
+        clone_dir: str
+):
+    if os.path.exists(clone_dir):
+        ori_dir = os.getcwd()
+        os.chdir(clone_dir)
+        os.system("GIT_LFS_SKIP_SMUDGE=1 git pull")
+        os.chdir(ori_dir)
+        message = f"the repo already exists, so just pull successfully!"
+    command = f"GIT_LFS_SKIP_SMUDGE=1 git clone https://oauth2:{ms_token}@www.modelscope.cn/datasets/{ms_repo_id}.git"
+    os.system(command)
+    configuration_path = os.path.join(clone_dir, "configuration.json")
+    if not os.path.exists(configuration_path):
+        open(configuration_path, "w").close()
+    # message
+    message = f"clone from https://oauth2:{ms_token}@www.modelscope.cn/datasets/{ms_repo_id}.git successfully!"
+    return message
+
+
+def hf_list_repo_files(
+        hf_token: str,
+        hf_repo_id: str,
+        repo_type: str = "dataset"
+):
+    hf_api = HfApi(token=hf_token)
+    files = hf_api.list_repo_files(repo_id=hf_repo_id, repo_type=repo_type)
+    return files
+
+
+def pull_from_hf(
+        hf_token: str,
+        hf_repo_id: str,
+        filename: str,
+):
+    if not hf_repo_id:
+        raise gr.Error("Please enter the repo_id of huggingface")
+    if not filename:
+        raise gr.Error("Please enter the filename")
+
+    if "," in filename:
+        filename_list = filename.split(",")
+        for _filename in filename_list:
+            save_path = os.path.join(LOCAL_DIR, _filename)
+            if os.path.exists(save_path):
+                message = "the file already exists!"
+                return message
+
+            # download
+            hf_api = HfApi(token=hf_token)
+            hf_api.hf_hub_download(
+                repo_id=hf_repo_id,
+                repo_type="dataset",
+                filename=_filename,
+                cache_dir=CACHE_DIR,
+                local_dir=LOCAL_DIR,
+                local_dir_use_symlinks=False
+            )
+    else:
+        save_path = os.path.join(LOCAL_DIR, filename)
+        if os.path.exists(save_path):
+            message = "the file already exists!"
+            return message
+
+        # download
+        hf_api = HfApi(token=hf_token)
+        hf_api.hf_hub_download(
+            repo_id=hf_repo_id,
+            repo_type="dataset",
+            filename=filename,
+            cache_dir=CACHE_DIR,
+            local_dir=LOCAL_DIR,
+            local_dir_use_symlinks=False
+        )
+
+    # message
+    message = f"Pull from https://huggingface.co/datasets/{hf_repo_id} successfully!"
+    print(message)
+    return message
+
+
+def move_file_from_local_to_clone_dir(
+        filename: str,
+        clone_dir: str
+):
+    # move to the clone dir
+    if "," in filename:
+        filename_list = filename.split(",")
+        for _filename in filename_list:
+            if "/" in _filename:
+                dirname = os.path.dirname(_filename)
+                src_dir = os.path.join(clone_dir, dirname)
+                if not os.path.exists(src_dir):
+                    os.makedirs(src_dir)
+            src = os.path.join(LOCAL_DIR, _filename)
+            dst = os.path.join(clone_dir, _filename)
+            shutil.move(src=src, dst=dst)
+    else:
+        if "/" in filename:
+            dirname = os.path.dirname(filename)
+            src_dir = os.path.join(clone_dir, dirname)
+            if not os.path.exists(src_dir):
+                os.makedirs(src_dir)
+        src = os.path.join(LOCAL_DIR, filename)
+        dst = os.path.join(clone_dir, filename)
+        shutil.move(src=src, dst=dst)
+
+    # message
+    message = f"move the file from {src} to {dst} successfully!"
+    print(message)
+    return message
+
+
+def push_to_ms(
+        username: str,
+        email: str,
+        ms_repo_id: str,
+        clone_dir: str,
+        filename: str
+):
+    # push to ms
+    ori_dir = os.getcwd()
+    os.chdir(clone_dir)
+    os.system("apt-get install git-lfs")
+    os.system("git lfs install")
+    os.system(f"git config --global user.email {email}")
+    os.system(f"git config --global user.name {username}")
+    if "," in filename:
+        filename_list = filename.split(",")
+        num = len(filename_list)
+        for _filename in filename_list:
+            os.system(f"git lfs track '{_filename}'")
+        os.system("git add .")
+        os.system(f"git commit -m 'upload {num} files'")
+        os.system(f"git push")
+        os.chdir(ori_dir)
+    else:
+        os.system(f"git lfs track '{filename}'")
+        os.system("git add .")
+        os.system(f"git commit -m 'upload {filename}'")
+        os.system(f"git push")
+        os.chdir(ori_dir)
+    # remove clone dir
+    if os.path.exists(clone_dir):
+        shutil.rmtree(clone_dir)
+    message = f'Pushed to https://www.modelscope.cn/datasets/{ms_repo_id} successfully!'
+    print(message)
+    return message
+
+
+def handle(
+        hf_token: str,
+        ms_token: str,
+        repo_type: str,
+        hf_repo: str,
+        ms_repo: str,
+):
+    clone_dir = ms_repo.split("/")[-1]
+    hf_file_list = hf_list_repo_files(hf_token, hf_repo, repo_type)
+    print(f"all file in hf: {hf_file_list}")
+
+    for filename in hf_file_list:
+        print(f"begin to process file: {filename}")
+        clone_from_ms(ms_token, ms_repo, clone_dir)
+        time.sleep(1)
+        pull_from_hf(hf_token, hf_repo, filename)
+        time.sleep(1)
+        move_file_from_local_to_clone_dir(filename, clone_dir)
+        time.sleep(1)
+        push_to_ms(USERNAME, EMAIL, ms_repo, clone_dir, filename)
+        print(f"process file finish: {filename}")
+        time.sleep(10)
+
+
+with gr.Blocks() as demo:
+    gr.Markdown(
+        '''
+        This space uploads model from Huggingface to ModelScope.
+        **Please make sure that you're the owner of the repo or have permission from the owner to do so!**
+        # How to use this Space?
+        - Duplicate this Space and providing MS token (optional) and your read/write HF token (mandatory)
+        - Create your target model repo on HF. This step needs to be done manually. The Space doesn't do create an empty repo for you.
+        - In your own private Space, fill in information below.
+        - Click submit then watch for output in container log for progress.
+        - Create README.md file (since the metadata is not compatible with HF)
+        '''
+    )
+    hf_token = gr.Textbox(label="HuggingFace Token")
+    ms_token = gr.Textbox(label="ModelScope Git Token")
+    repo_type = gr.Textbox(label="Repo Type", value="dataset")
+    hf_repo = gr.Textbox(label="HuggingFace Repo")
+    ms_repo = gr.Textbox(label="ModelScope Repo")
+
+    with gr.Row():
+        button = gr.Button("Submit", variant="primary")
+        clear = gr.Button("Clear")
+
+    button.click(
+        handle,
+        [hf_token, ms_token, repo_type, hf_repo, ms_repo],
+        outputs=None
+    )
+
+if __name__ == "__main__":
+    demo.queue(max_size=1)
+    demo.launch(share=False, max_threads=1)