Upload folder using huggingface_hub

app/draw_diagram.py

@@ -0,0 +1,157 @@
+import streamlit as st
+import pandas as pd
+import numpy as np
+from streamlit_echarts import st_echarts
+# from streamlit_echarts import JsCode
+from streamlit_javascript import st_javascript
+# from PIL import Image 
+
+links_dic = {"random": "https://seaeval.github.io/", 
+             "meta_llama_3_8b": "https://huggingface.co/meta-llama/Meta-Llama-3-8B", 
+             "mistral_7b_instruct_v0_2": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2", 
+             "sailor_0_5b": "https://huggingface.co/sail/Sailor-0.5B", 
+             "sailor_1_8b": "https://huggingface.co/sail/Sailor-1.8B", 
+             "sailor_4b": "https://huggingface.co/sail/Sailor-4B", 
+             "sailor_7b": "https://huggingface.co/sail/Sailor-7B", 
+             "sailor_0_5b_chat": "https://huggingface.co/sail/Sailor-0.5B-Chat", 
+             "sailor_1_8b_chat": "https://huggingface.co/sail/Sailor-1.8B-Chat", 
+             "sailor_4b_chat": "https://huggingface.co/sail/Sailor-4B-Chat", 
+             "sailor_7b_chat": "https://huggingface.co/sail/Sailor-7B-Chat", 
+             "sea_mistral_highest_acc_inst_7b": "https://seaeval.github.io/", 
+             "meta_llama_3_8b_instruct": "https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct", 
+             "flan_t5_base": "https://huggingface.co/google/flan-t5-base", 
+             "flan_t5_large": "https://huggingface.co/google/flan-t5-large", 
+             "flan_t5_xl": "https://huggingface.co/google/flan-t5-xl", 
+             "flan_t5_xxl": "https://huggingface.co/google/flan-t5-xxl", 
+             "flan_ul2": "https://huggingface.co/google/flan-t5-ul2", 
+             "flan_t5_small": "https://huggingface.co/google/flan-t5-small", 
+             "mt0_xxl": "https://huggingface.co/bigscience/mt0-xxl", 
+             "seallm_7b_v2": "https://huggingface.co/SeaLLMs/SeaLLM-7B-v2", 
+             "gpt_35_turbo_1106": "https://openai.com/blog/chatgpt", 
+             "meta_llama_3_70b": "https://huggingface.co/meta-llama/Meta-Llama-3-70B", 
+             "meta_llama_3_70b_instruct": "https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct", 
+             "sea_lion_3b": "https://huggingface.co/aisingapore/sea-lion-3b", 
+             "sea_lion_7b": "https://huggingface.co/aisingapore/sea-lion-7b", 
+             "qwen1_5_110b": "https://huggingface.co/Qwen/Qwen1.5-110B", 
+             "qwen1_5_110b_chat": "https://huggingface.co/Qwen/Qwen1.5-110B-Chat", 
+             "llama_2_7b_chat": "https://huggingface.co/meta-llama/Llama-2-7b-chat-hf", 
+             "gpt4_1106_preview": "https://openai.com/blog/chatgpt", 
+             "gemma_2b": "https://huggingface.co/google/gemma-2b", 
+             "gemma_7b": "https://huggingface.co/google/gemma-7b", 
+             "gemma_2b_it": "https://huggingface.co/google/gemma-2b-it", 
+             "gemma_7b_it": "https://huggingface.co/google/gemma-7b-it", 
+             "qwen_1_5_7b": "https://huggingface.co/Qwen/Qwen1.5-7B", 
+             "qwen_1_5_7b_chat": "https://huggingface.co/Qwen/Qwen1.5-7B-Chat", 
+             "sea_lion_7b_instruct": "https://huggingface.co/aisingapore/sea-lion-7b-instruct", 
+             "sea_lion_7b_instruct_research": "https://huggingface.co/aisingapore/sea-lion-7b-instruct-research", 
+             "LLaMA_3_Merlion_8B": "https://seaeval.github.io/", 
+             "LLaMA_3_Merlion_8B_v1_1": "https://seaeval.github.io/"}
+
+links_dic = {k.lower().replace('_', '-') : v for k, v in links_dic.items()}
+
+# huggingface_image = Image.open('style/huggingface.jpg')
+
+def nav_to(value):
+    try:
+        url = links_dic[str(value).lower()]
+        js = f'window.open("{url}", "_blank").then(r => window.parent.location.href);'
+        st_javascript(js)
+    except:
+        pass
+
+def draw(folder_name,category_name, dataset_name, sorted):
+    
+    folder = f"./results/{folder_name}/"
+
+    display_names = {
+        'ASR': 'Automatic Speech Recognition',
+        'SQA': 'Speech Question Answering',
+        'SI': 'Speech Instruction',
+        'AC': 'Audio Captioning',
+        'ASQA': 'Audio Scene Question Answering',
+        'AR': 'Accent Recognition',
+        'GR': 'Gender Recognition',
+        'ER': 'Emotion Recognition'
+    }
+    
+    data_path = f'{folder}/{category_name.lower()}.csv'
+    chart_data = pd.read_csv(data_path).round(2).dropna(axis=0)
+
+    if len(chart_data) == 0:
+        return
+
+
+    if sorted == 'Ascending':
+        ascend = True 
+    else:
+        ascend = False
+
+    sort_by = dataset_name.replace('-', '_').lower()
+    
+    chart_data = chart_data.sort_values(by=[sort_by], ascending=ascend)
+    
+    min_value = round(chart_data.iloc[:, 1::].min().min() - 0.1, 1) 
+    max_value = round(chart_data.iloc[:, 1::].max().max() + 0.1, 1) 
+
+    columns = list(chart_data.columns)[1:]
+    series = []
+    for col in columns:
+        series.append(
+            {
+                "name": f"{col.replace('_', '-')}",
+                "type": "line",
+                "data": chart_data[f'{col}'].tolist(),
+            }
+            )
+        
+
+    options = {
+        "title": {"text": f"{display_names[category_name]}"},
+        "tooltip": {
+            "trigger": "axis",
+            "axisPointer": {"type": "cross", "label": {"backgroundColor": "#6a7985"}},
+            "triggerOn": 'mousemove',
+        },
+        "legend": {"data": ['Overall Accuracy']},
+        "toolbox": {"feature": {"saveAsImage": {}}},
+        "grid": {"left": "3%", "right": "4%", "bottom": "3%", "containLabel": True},
+        "xAxis": [
+            {
+                "type": "category",
+                "boundaryGap": False,
+                "triggerEvent": True,
+                "data": chart_data['Model'].tolist(),
+            }
+        ],
+        "yAxis": [{"type": "value", 
+                    "min": min_value,
+                    "max": max_value, 
+                    # "splitNumber": 10
+                    }],
+        "series": series,
+    }
+    
+    events = {
+        "click": "function(params) { return params.value }"
+    }
+
+    value = st_echarts(options=options, events=events, height="500px")
+    
+    if value != None:
+        # print(value)
+        nav_to(value)
+
+    # if value != None:
+    #     highlight_table_line(value)
+
+    ### create table
+    st.divider()
+    # chart_data['Link'] = chart_data['Model'].map(links_dic)
+    st.dataframe(chart_data,
+                #  column_config = {
+                #      "Link": st.column_config.LinkColumn(
+                #          display_text= st.image(huggingface_image)
+                #      ),
+                #  }, 
+                    hide_index = True, 
+                    use_container_width=True)

app/pages.py

@@ -0,0 +1,160 @@
+import streamlit as st
+from app.draw_diagram import *
+
+def dashboard():
+
+    with st.container():
+        st.title("AudioBench")
+   
+        st.markdown("""
+            [gh]: https://github.com/AudioLLMs/AudioBench
+            [![GitHub watchers](https://img.shields.io/github/watchers/AudioLLMs/AudioBench?style=social)][gh]
+            [![GitHub Repo stars](https://img.shields.io/github/stars/AudioLLMs/AudioBench?style=social)][gh]
+            """)
+
+    audio_url = "https://arxiv.org/abs/2406.16020"
+
+    st.divider()
+    st.markdown("#### [AudioBench](%s)" % audio_url)
+    st.markdown("##### :dizzy: A comprehensive evaluation benchmark designed for general instruction-following audiolanguage models")
+    st.markdown('''
+
+                
+                ''')
+
+    with st.container():
+        left_co, center_co, right_co = st.columns([0.5,1, 0.5])
+        with center_co:
+            st.image("./style/audio_overview.png", 
+                     caption="Overview of the datasets in AudioBench.", 
+                     use_column_width = True)
+        
+        st.markdown('''
+
+                
+                ''')
+        
+        st.markdown("###### :dart: Our Benchmark includes: ")
+        cols = st.columns(10)
+        cols[1].metric(label="Tasks", value="8", delta="Tasks", delta_color="off")
+        cols[2].metric(label="Datasets", value="26", delta="Datasets", delta_color="off")
+        cols[3].metric(label="Test On", value="4", delta="Models", delta_color="off")
+
+        # st.markdown("###### :dart: Supported Models and Datasets: ")
+        
+        # sup = pd.DataFrame(
+        #         {"Dataset": "LibriSpeech-Clean", 
+        #          "Category": st.selectbox('category', ['Speech Understanding']),
+        #          "Task": st.selectbox('task', ['Automatic Speech Recognition']),
+        #          "Metrics": st.selectbox('metrics', ['WER']),
+        #          "Status":True}
+        # )
+        
+        # st.data_editor(sup, num_rows="dynamic")
+
+
+
+    st.divider()
+    with st.container():
+        st.markdown("##### Citations")
+
+        st.markdown('''
+                    :round_pushpin: AudioBench Paper \n
+                        @article{wang2024audiobench,
+                            title={AudioBench: A Universal Benchmark for Audio Large Language Models},
+                            author={Wang, Bin and Zou, Xunlong and Lin, Geyu and Sun, Shuo and Liu, Zhuohan and Zhang, Wenyu and Liu, Zhengyuan and Aw, AiTi and Chen, Nancy F},
+                            journal={arXiv preprint arXiv:2406.16020},
+                            year={2024}
+                            }
+                    ''')
+
+def speech_understanding():
+    st.title("Speech Understanding")
+    
+    filters_levelone = ['ASR', 'SQA', 'SI']
+    sort_leveltwo = []
+    
+    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
+    
+    with left:
+        filter_1 = st.selectbox('Select Category', filters_levelone)
+    
+    with middle:
+        if filter_1 == filters_levelone[0]:
+            sort_leveltwo = ['LibriSpeech-Test-Clean', 'LibriSpeech-Test-Other', 'Common-Voice-15-En-Test', 'Peoples-Speech-Test',
+                            'GigaSpeech-Test', 'Tedlium3-Test','Tedlium3-Longform-Test', 'Earning-21-Test', 'Earning-22-Test']
+        elif filter_1 == filters_levelone[1]:
+            sort_leveltwo = ['CN-College-Listen-Test', 'SLUE-P2-SQA5-Test', 'DREAM-TTS-Test', 'Public-SG-SpeechQA-Test']
+        
+        elif filter_1 == filters_levelone[2]:
+            sort_leveltwo = ['OpenHermes-Audio-Test', 'ALPACA-Audio-Test']
+        
+        sort = st.selectbox("Sort Dataset", sort_leveltwo)
+    
+    with right:
+        sorted = st.selectbox('by', ['Ascending', 'Descending'])
+
+    if filter_1 or sort or sorted:
+        draw('su',filter_1, sort, sorted)
+    else:
+        draw('su', 'ASR', 'LibriSpeech-Test-Clean', 'Descending')
+
+
+def audio_scene_understanding():
+    st.title("Audio Scence Understanding")
+
+    filters_levelone = ['AQA', 'AC']
+    sort_leveltwo = []
+    
+    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
+    
+    with left:
+        filter_1 = st.selectbox('Select Category', filters_levelone)
+    
+    with middle:
+        if filter_1 == filters_levelone[0]:
+            sort_leveltwo = ['Clotho-AQA-Test', 'WavCaps-QA-Test', 'AudioCaps-QA-Test']
+        elif filter_1 == filters_levelone[1]:
+            sort_leveltwo = ['WavCaps-Test', 'AudioCaps-Test']
+        
+        sort = st.selectbox("Sort Dataset", sort_leveltwo)
+    
+    with right:
+        sorted = st.selectbox('by', ['Ascending', 'Descending'])
+
+    if filter_1 or sort or sorted:
+        draw('asu',filter_1, sort, sorted)
+    else:
+        draw('asu', 'AQA', 'Clotho-AQA-Test', 'Descending')
+
+
+def voice_understanding():
+    st.title("Voice Understanding")
+
+    filters_levelone = ['ER', 'AR', 'GR']
+    sort_leveltwo = []
+    
+    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
+    
+    with left:
+        filter_1 = st.selectbox('Select Category', filters_levelone)
+    
+    with middle:
+        if filter_1 == filters_levelone[0]:
+            sort_leveltwo = ['IEMOCAP-Emotion-Test', 'MELD-Sentiment-Test', 'MELD-Emotion-Test']
+        
+        elif filter_1 == filters_levelone[1]:
+            sort_leveltwo = ['VoxCeleb1-Accent-Test']
+        
+        elif filter_1 == filters_levelone[2]:
+            sort_leveltwo = ['VoxCeleb1-Gender-Test', 'IEMOCAP-Gender-Test']
+        
+        sort = st.selectbox("Sort Dataset", sort_leveltwo)
+    
+    with right:
+        sorted = st.selectbox('by', ['Ascending', 'Descending'])
+
+    if filter_1 or sort or sorted:
+        draw('vu',filter_1, sort, sorted)
+    else:
+        draw('vu', 'ER', 'IEMOCAP-Emotion-Test', 'Descending')