updated to v0.0.4

Dashboard.py

@@ -47,14 +47,20 @@ with st.expander("Prompt downloader"):
 
     # Concat all tasks to dataframe
     prompt_download = pd.concat(prompt_download_dict.values())
+    # Exclude prompts from single object prompt download, as else the int transform gives an error
+    single_object_prompt_download = prompt_download.dropna(subset='Linked_prompts')
 
     # Add relevant single object prompts
-    single_object_ids = prompt_download.Linked_prompts.str.split(',').explode().unique().astype('int')
+    single_object_ids = single_object_prompt_download.Linked_prompts.str.split(',').explode().unique().astype('int')
     prompt_download = pd.concat([
        prompt_download,
        prompt_dir.loc[prompt_dir['ID'].isin(single_object_ids)] 
     ])
 
+    # For img2img prompt, the prompt in the download gets replaced by img2img instructions
+    img2img_instructions_col = prompt_download.loc[prompt_download['Task'].str.startswith('img2img')]['img2img_instructions']
+    prompt_download.loc[prompt_download['Task'].str.startswith('img2img'),'Prompt']=img2img_instructions_col
+
     # Add download button for prompts
     st.download_button(
         label="Download prompts",
@@ -140,5 +146,9 @@ if eval_df.shape[0]!=0:
     st.write("- Available for manual assessment: ", str(sum(eval_df.manual_eval)))
     manual_eval_available = sum(eval_df.manual_eval)
     st.write("- Available for automated assessment: ", str(sum(eval_df.automated_eval)))
+
+    if eval_df.shape[0]>sum(eval_df.manual_eval):
+        st.write('WARNING: {0} image(s) with invalid file names uploaded. Pictures with invalid names will not be available for assessment. Use the file names provided by the prompt downloader to correctly name your generated images.'.format(str(eval_df.shape[0]-sum(eval_df.manual_eval))))
+
 else:
     st.write("Upload files to start the assessment.")

Dashboard_setup.py

@@ -1,4 +1,4 @@
 import pandas as pd 
 
 automated_task_list = ['Multiple object types', 'Single object','Negation']
-prompt_dir = pd.read_csv('Data/Prompt_dir_221216.csv')
+prompt_dir = pd.read_csv('Data/Prompt_dir_230104.csv')

README.md

@@ -10,4 +10,4 @@ pinned: false
 license: mit
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+We provide a version of local hosting with a customization guide on https://github.com/8erberg/Intel-Generative-Image-Dashboard-experimental

pages/1_⚙️Manual assessment.py

@@ -2,7 +2,8 @@ import streamlit as st
 import numpy as np
 import pandas as pd
 from PIL import Image
-from pages.Functions.Dashboard_functions import add_previous_manual_assessments
+from pages.Functions.Dashboard_functions import add_previous_manual_assessments, delete_last_manual_rating
+
 
 st.title('Manual assessment')
 st.write('On this page you can rate all uploaded images with regards to how good they match their respective prompts. You can see the outcome of your assessment on the summary page.')
@@ -26,6 +27,14 @@ except KeyError:
     manual_eval_available = 0
     st.session_state['uploaded_img'] = [] #safety if program is started on manual assesssment page and not desktop
 
+# Create manual rating history if it does not already exist
+try:
+    _ = st.session_state['manual_rating_history'][-1]
+except KeyError:
+    st.session_state['manual_rating_history'] = []
+except IndexError:
+    pass
+
 
 # Main rating loop
 ## If images are available for rating this creates a from to submit ratings to database
@@ -113,6 +122,9 @@ if manual_eval_available > 0:
         # Submit assessments to database
         submitted = st.form_submit_button("Submit")
         if submitted:
+            # Create temporary list to hold picture indexes for this run
+            temp_picture_index_list = []
+
             # First add main prompt assessment
             st.session_state['eval_df'].loc[
                 curr_picture_index,'manual_eval']=include_prompt
@@ -121,6 +133,9 @@ if manual_eval_available > 0:
             st.session_state['eval_df'].loc[
                 curr_picture_index,'manual_eval_task_score']=curr_manual_eval_row['manual_eval_task_score'].item()       
 
+            # Add picture index to temp list
+            temp_picture_index_list.append(curr_picture_index)
+
             # Add subprompt assessment if dataset was created for subprompts
             # This stage will automatically be skipped if the df for linked prompts is empty
             for row in curr_linked_rows.itertuples():
@@ -131,9 +146,18 @@ if manual_eval_available > 0:
                 st.session_state['eval_df'].loc[
                     row.Picture_index,'manual_eval_task_score']=row.manual_eval_task_score
 
+                # Add picture index to temp list
+                temp_picture_index_list.append(row.Picture_index)
+
+            # Add temp list of picture indices to rating history
+            st.session_state['manual_rating_history'].append(temp_picture_index_list)
+
             # Reset page after ratings were submitted
             st.experimental_rerun()
 
+    # Return to last rated image
+    delete_last_manual_rating()
+
     # Add option to add previous manual assessments
     add_previous_manual_assessments()
 
@@ -143,7 +167,6 @@ elif len(st.session_state['uploaded_img'])==0:
 # If files are uploaded but all ratings are completed
 else:
     assessment_progress.write('You finished assessing the current batch of uploaded images. Upload more pictures of generate your results on the summary page.')
-
-
-
+    # Add option to return to last manual rating
+    delete_last_manual_rating()
 

pages/2_🤖Automated assessment.py

@@ -40,7 +40,7 @@ if automated_eval_available > 0:
 
     with st.form("auto_assessment_form",clear_on_submit=True):
         # Form info statment
-        st.write('Select tasks to assess with the automated assessment:')
+        st.write('Select tasks to assess with the automated assessment below. Once you started an assessment you will not be able to leave this page before the assessment is completed.')
 
         # Create list of bool selection buttons, one for every task
         for i_task in range(task_list_len):

pages/3_📊Assessment summary.py

@@ -3,10 +3,11 @@ import pandas as pd
 import seaborn as sns
 import matplotlib.pyplot as plt
 from PIL import Image
-from pages.Functions.Dashboard_functions import plot_style_simple, plot_style_combined, print_results_tabs, pre_assessment_visualisation
+from pages.Functions.Dashboard_functions import pre_assessment_visualisation, multi_comparison_plotI, print_results_tabs
 side_image = Image.open('Graphics/IL_Logo.png')
 st.sidebar.image(side_image)
 
+
 @st.cache
 def convert_df_to_csv(df):
   # IMPORTANT: Cache the conversion to prevent computation on every rerun
@@ -18,26 +19,22 @@ assessment_result_frames = {}
 st.title('Assessment Summary')
 st.header('Manual assessment')
 
+
 try:
   if sum(st.session_state['eval_df']['manual_eval_completed'])>0:
     # Display file uploader
-    manual_file_upload = st.file_uploader("Upload .csv with saved manual assessment for model comparison")
-
+    manual_file_upload = st.file_uploader("Upload .csv with saved manual assessment for model comparison", accept_multiple_files=True)
     # Create dataset for manual summary plots
     manual_eval_df = st.session_state['eval_df']
     manual_eval_df['Score'] = manual_eval_df['manual_eval_task_score'].map({'Yes':True, 'No':False})
     manual_results_df = manual_eval_df.loc[
       (manual_eval_df['manual_eval']==True)&
       (manual_eval_df['manual_eval_completed']==True)]
-
+    manual_results_df['Model']='Manual assessment'
     assessment_result_frames['Manual assessment'] = manual_results_df
 
     # Add plots / tables to page
-    try:
-      manual_file_upload_df = pd.read_csv(manual_file_upload).copy()
-      print_results_tabs(file_upload=manual_file_upload, results_df=manual_results_df, file_upload_df=manual_file_upload_df)
-    except ValueError:
-      print_results_tabs(file_upload=manual_file_upload, results_df=manual_results_df)
+    print_results_tabs(file_upload=manual_file_upload, results_df=manual_results_df)
 
     st.download_button(
       label="Download manual assessment data",
@@ -50,24 +47,19 @@ try:
 except KeyError:
   pre_assessment_visualisation(type_str='manual')
 
-
-
 st.write(' ')
 st.header('Automated assessment')
 try:
   # Create dataset for automated summary plots
   auto_eval_df = st.session_state['auto_eval_df']
+  auto_eval_df['Model']='Automated assessment'
   assessment_result_frames['Automated assessment'] = auto_eval_df
 
   # Display file uploader
-  auto_file_upload = st.file_uploader("Upload .csv with saved automated assessment for model comparison")  
+  auto_file_upload = st.file_uploader("Upload .csv with saved automated assessment for model comparison", accept_multiple_files=True)  
 
   # Add plots / tables to page
-  try:
-    auto_file_upload_df = pd.read_csv(auto_file_upload).copy()
-    print_results_tabs(file_upload=auto_file_upload, results_df=auto_eval_df, file_upload_df=auto_file_upload_df)
-  except ValueError:
-    print_results_tabs(file_upload=auto_file_upload, results_df=auto_eval_df)
+  print_results_tabs(file_upload=auto_file_upload, results_df=auto_eval_df)
 
   st.download_button(
     label="Download automated assessment data",
@@ -121,3 +113,4 @@ except IndexError:
   st.write('There is no image availabe in your selected category.')
 except KeyError:
   pass
+

pages/Functions/Dashboard_functions.py

@@ -2,10 +2,37 @@
 
 import streamlit as st
 import pandas as pd
+import numpy as np
 import seaborn as sns
 import matplotlib.pyplot as plt
 from PIL import Image
 
+##### Page-unspecific functions
+
+def assert_uploaded_frame(uploaded_df):
+    # Set up variables checked for
+    asserted_columns = {
+      'Prompt_no':pd.api.types.is_integer_dtype,
+      'Score':pd.api.types.is_bool_dtype,
+      'Task':pd.api.types.is_object_dtype,
+      'File_name':pd.api.types.is_object_dtype}
+    asserted_column_names = ['Prompt_no','Score','Task','File_name']
+
+    # Check whether all needed column names are present
+    existing_column_names = [(x in uploaded_df.columns) for x in asserted_column_names]
+    assert all(existing_column_names), "The uploaded dataframe is missing a column needed for import. Your table needs to contain the columns: 'Prompt_no', 'Score', 'Task', 'File_name' "
+
+    # Check whether all needed columns have correct dtypes
+    correct_column_dtypes = []
+    for i_item in asserted_columns.items():
+        dtype_test = i_item[1](uploaded_df[i_item[0]].dtype)
+        correct_column_dtypes.append(dtype_test)
+    assert all(correct_column_dtypes), "Incorrect dtypes in uploaded dataframe."
+
+def assert_multi_frame_upload(list_of_uploaded_dfs):
+    # Apply uploaded frame assert to list of frames
+    for i_df in list_of_uploaded_dfs:
+        assert_uploaded_frame(i_df)
 
 ##### Dashboard main page
 def prompt_to_csv(df):
@@ -16,6 +43,22 @@ def prompt_to_csv(df):
 
 ##### Manual assessment
 
+def delete_last_manual_rating():
+    '''
+    Routine to delete last manual rating and hence to return to it
+    '''
+    if len(st.session_state['manual_rating_history'])>0:
+
+        if st.button('Return to last rated image'):
+            # The list contains sublists of images rated together, here we loop over these images to reset all of them
+            deleted_picture_index_list = st.session_state['manual_rating_history'].pop()
+            for i_picind in deleted_picture_index_list:
+                st.session_state['eval_df'].loc[
+                    i_picind,'manual_eval_completed']=False
+                st.session_state['eval_df'].loc[
+                    i_picind,'manual_eval_task_score']=np.nan  
+            st.experimental_rerun() 
+
 def add_previous_manual_assessments():
     '''
     This is a routine to allow the user to upload prior manual ratings and override
@@ -31,6 +74,11 @@ def add_previous_manual_assessments():
     if uploaded_ratings != None:
         try:
             uploaded_ratings_df = pd.read_csv(uploaded_ratings)
+            
+            # Run standard assert pipeline
+            assert_uploaded_frame(uploaded_ratings_df)
+
+            # Show matching image count and instructions
             overlapping_files_df =pd.merge(st.session_state['eval_df'],uploaded_ratings_df,on='File_name',how='inner')
             st.write('Number of matching file names found: '+ str(len(overlapping_files_df)))
             st.write('Click "Add results" button to add / override current ratings with uploaded ratings.')
@@ -56,6 +104,75 @@ def add_previous_manual_assessments():
 
 
 ##### Assessment summary
+
+def print_results_tabs(file_upload, results_df):
+    '''
+    #Routine used to give user the choice between showing results as bar chart or table
+    '''
+    # Create a tab for bar chart and one for table data
+    fig, table = multi_comparison_plotI(results_df=results_df, uploaded_df_list=file_upload)
+    tab1, tab2 = st.tabs(["Bar chart", "Data table"])
+    with tab1:
+      st.pyplot(fig)
+
+    with tab2:
+        st.write(table)
+
+
+def pre_assessment_visualisation(type_str):
+    '''
+    Routine used to allow user to visualise uploaded results before completing any assessments
+    '''
+    st.write('Complete {0} assessment or upload .csv with saved {0} assessment to generate summary.'.format(type_str))
+
+    # Display file uploader
+    file_upload = st.file_uploader("Upload .csv with saved {0} assessment to plot prior results.".format(type_str), accept_multiple_files=True)
+    if len(file_upload) > 0:
+        print_results_tabs(file_upload=file_upload, results_df=None)
+
+
+def multi_comparison_plotI(results_df = None, uploaded_df_list = []):
+    # If list of uploaded_dfs is provided and we transform them into pd.Dfs
+    # Multiple file uploader returns empty list as default
+    file_upload_names = [x.name for x in uploaded_df_list]
+    plot_df_list = [pd.read_csv(x) for x in uploaded_df_list]
+
+    # Assert that all uploaded df's have correct format
+    assert_multi_frame_upload(plot_df_list)
+
+    # Add file name as model name
+    for i_df in range(len(file_upload_names)):
+        plot_df_list[i_df]= plot_df_list[i_df].assign(Model=file_upload_names[i_df])
+
+    # If results df is provided, add it to list of dfs to plot
+    if type(results_df) == pd.DataFrame:
+        plot_df_list.append(results_df)
+
+    # Concat all frames to joined dataframe
+    plot_df = pd.concat(plot_df_list)
+
+    # Calculate the grouped percentage scores per task category and model
+    grouped_series = plot_df.groupby(['Task','Model'])['Score'].sum()/plot_df.groupby(['Task','Model'])['Score'].count()*100
+    grouped_series = grouped_series.rename('Percentage correct')
+
+    # Create plot
+    eval_share = grouped_series.reset_index()
+    # Add small amount to make the bars on plot not disappear
+    eval_share['Percentage correct'] = eval_share['Percentage correct']+1
+
+    # Create plot
+    fig = plt.figure(figsize=(12, 3))
+    sns.barplot(data=eval_share,x='Task',y='Percentage correct',hue='Model', palette='GnBu')
+    plt.xticks(rotation=-65)
+    plt.xlabel(' ')
+    plt.ylim(0, 100)
+    return fig,grouped_series
+
+
+
+
+############## Functions no longer used, to be deleted
+
 def plot_style_simple(results_df, return_table = False):
     '''
     Simple plot function for plotting just one dataframe of results
@@ -109,42 +226,4 @@ def plot_style_combined(results_df, uploaded_df = None, return_table=False):
     plt.xticks(rotation=-65)
     plt.ylabel('Percentage correct')
     plt.xlabel(' ')
-    return fig
-
-
-def print_results_tabs(file_upload, results_df, file_upload_df=None):
-    '''
-    Routine used to give user the choice between showing results as bar chart or table
-    '''
-    # Create a tab for bar chart and one for table data
-    tab1, tab2 = st.tabs(["Bar chart", "Data table"])
-    with tab1:
-        # If df was uploaded for comparison, we create comparison plot, else simple plot
-        if file_upload == None:
-            fig = plot_style_simple(results_df)
-            st.pyplot(fig)
-        else:
-            fig = plot_style_combined(results_df,file_upload_df)
-            st.pyplot(fig)
-
-    with tab2:
-        # If df was uploaded for comparison, we create comparison table, else simple table
-        if file_upload == None:
-            table = plot_style_simple(results_df, return_table=True)
-            st.write(table)
-        else:
-            table = plot_style_combined(results_df,file_upload_df, return_table=True)
-            st.write(table)
-
-
-def pre_assessment_visualisation(type_str):
-    '''
-    Routine used to allow user to visualise uploaded results before completing any assessments
-    '''
-    st.write('Complete {0} assessment or upload .csv with saved {0} assessment to generate summary.'.format(type_str))
-
-    # Display file uploader
-    file_upload = st.file_uploader("Upload .csv with saved {0} assessment to plot prior results.".format(type_str))
-    if file_upload != None:
-        file_upload_df = pd.read_csv(file_upload).copy()
-        print_results_tabs(file_upload=None, results_df=file_upload_df)
+    return fig