clefourrier HF staff derek-thomas HF staff commited on
Commit
95c19d6
1 Parent(s): 6bc96ff

pr_3_add_button_readme_and_better_logs (#3)

Browse files

- Adding better readme (6a9c17dcb2c0970fff46a5f1fba87c2d4e1bd848)
- Adding execute button (80a20422e63f5db2430aa8780d08df11bd7402bf)
- Adding better logs (b2a9c221b75f4c63c7e74c5c56eb81aba74b93d2)
- Adding better exception handling (8bd884a7f207ef9cec1c613edc89a9ff1725c843)
- Adding reverse order (d38510d004f1574318e214adbc143776a71309d0)
- Restructuring and adding download button (f59a2e9cf0439a833071d6f47dfdce948d2087de)
- Ignore logfile (b149c0db43c5eba0158f0c392efea52e40b2d8c3)


Co-authored-by: Derek Thomas <[email protected]>

.gitignore CHANGED
@@ -12,3 +12,4 @@ eval-results/
12
  eval-queue-bk/
13
  eval-results-bk/
14
  logs/
 
 
12
  eval-queue-bk/
13
  eval-results-bk/
14
  logs/
15
+ output.log
app.py CHANGED
@@ -1,26 +1,60 @@
1
  import logging
2
- import sys
 
 
 
 
 
 
3
 
4
  import gradio as gr
5
  from main_backend_lighteval import run_auto_eval
6
  from src.display.log_visualizer import log_file_to_html_string
7
  from src.display.css_html_js import dark_mode_gradio_js
8
- from src.envs import REFRESH_RATE
 
9
 
10
  logging.basicConfig(level=logging.INFO)
 
11
 
12
 
13
  intro_md = f"""
14
  # Intro
15
- This is just a visual for the auto evaluator. Note that the lines of the log visual are reversed.
16
- # Logs
 
 
 
 
 
 
 
 
17
  """
18
 
 
 
 
 
 
 
 
19
  with gr.Blocks(js=dark_mode_gradio_js) as demo:
 
20
  with gr.Tab("Application"):
21
- gr.Markdown(intro_md)
22
- output = gr.HTML(log_file_to_html_string, every=10)
 
 
 
 
 
 
 
23
  dummy = gr.Markdown(run_auto_eval, every=REFRESH_RATE, visible=False)
24
 
 
 
 
25
  if __name__ == '__main__':
26
  demo.queue(default_concurrency_limit=40).launch(server_name="0.0.0.0", show_error=True, server_port=7860)
 
1
  import logging
2
+ from src.logging import configure_root_logger
3
+ logging.getLogger("httpx").setLevel(logging.WARNING)
4
+ logging.getLogger("numexpr").setLevel(logging.WARNING)
5
+ logging.getLogger("absl").setLevel(logging.WARNING)
6
+ configure_root_logger()
7
+
8
+ from functools import partial
9
 
10
  import gradio as gr
11
  from main_backend_lighteval import run_auto_eval
12
  from src.display.log_visualizer import log_file_to_html_string
13
  from src.display.css_html_js import dark_mode_gradio_js
14
+ from src.envs import REFRESH_RATE, REPO_ID, QUEUE_REPO, RESULTS_REPO
15
+ from src.logging import setup_logger, log_file
16
 
17
  logging.basicConfig(level=logging.INFO)
18
+ logger = setup_logger(__name__)
19
 
20
 
21
  intro_md = f"""
22
  # Intro
23
+ This is a visual for the auto evaluator.
24
+ """
25
+
26
+ links_md = f"""
27
+ # Important links
28
+ | Description | Link |
29
+ |-----------------|------|
30
+ | Leaderboard | [{REPO_ID}](https://huggingface.co/spaces/{REPO_ID}) |
31
+ | Queue Repo | [{QUEUE_REPO}](https://huggingface.co/datasets/{QUEUE_REPO}) |
32
+ | Results Repo | [{RESULTS_REPO}](https://huggingface.co/datasets/{RESULTS_REPO}) |
33
  """
34
 
35
+ def button_auto_eval():
36
+ logger.info("Manually triggering Auto Eval")
37
+ run_auto_eval()
38
+
39
+
40
+ reverse_order_checkbox = gr.Checkbox(label="Reverse Order", value=True)
41
+
42
  with gr.Blocks(js=dark_mode_gradio_js) as demo:
43
+ gr.Markdown(intro_md)
44
  with gr.Tab("Application"):
45
+ output_html = gr.HTML(partial(log_file_to_html_string, reverse=reverse_order_checkbox), every=1)
46
+ with gr.Row():
47
+ download_button = gr.DownloadButton("Download Log File", value=log_file)
48
+ with gr.Accordion('Log View Configuration', open=False):
49
+ reverse_order_checkbox.render()
50
+ # Add a button that when pressed, triggers run_auto_eval
51
+ button = gr.Button("Manually Run Evaluation")
52
+ gr.Markdown(links_md)
53
+
54
  dummy = gr.Markdown(run_auto_eval, every=REFRESH_RATE, visible=False)
55
 
56
+ button.click(fn=button_auto_eval, inputs=[], outputs=[])
57
+
58
+
59
  if __name__ == '__main__':
60
  demo.queue(default_concurrency_limit=40).launch(server_name="0.0.0.0", show_error=True, server_port=7860)
main_backend_lighteval.py CHANGED
@@ -70,6 +70,7 @@ def run_auto_eval():
70
  # instance_size, instance_type = "small", "g4dn.xlarge"
71
  # For CPU
72
  instance_size, instance_type = "medium", "c6i"
 
73
 
74
  run_evaluation(
75
  eval_request=eval_request,
@@ -84,6 +85,8 @@ def run_auto_eval():
84
  limit=LIMIT
85
  )
86
 
 
 
87
 
88
  if __name__ == "__main__":
89
  run_auto_eval()
 
70
  # instance_size, instance_type = "small", "g4dn.xlarge"
71
  # For CPU
72
  instance_size, instance_type = "medium", "c6i"
73
+ logger.info(f'Starting Evaluation of {eval_request.json_filepath} on Inference endpoints: {instance_size} {instance_type}')
74
 
75
  run_evaluation(
76
  eval_request=eval_request,
 
85
  limit=LIMIT
86
  )
87
 
88
+ logger.info(f'Completed Evaluation of {eval_request.json_filepath} on Inference endpoints: {instance_size} {instance_type}')
89
+
90
 
91
  if __name__ == "__main__":
92
  run_auto_eval()
src/backend/run_eval_suite_lighteval.py CHANGED
@@ -61,7 +61,7 @@ def run_evaluation(eval_request: EvalRequest, task_names: str, batch_size: int,
61
 
62
  dumped = json.dumps(results, indent=2)
63
  logger.info(dumped)
64
- except Exception: # if eval failed, we force a cleanup
65
  env_config = EnvConfig(token=TOKEN, cache_dir=args.cache_dir)
66
 
67
  model_config = create_model_config(args=args, accelerator=accelerator)
 
61
 
62
  dumped = json.dumps(results, indent=2)
63
  logger.info(dumped)
64
+ except Exception as e: # if eval failed, we force a cleanup
65
  env_config = EnvConfig(token=TOKEN, cache_dir=args.cache_dir)
66
 
67
  model_config = create_model_config(args=args, accelerator=accelerator)
src/display/log_visualizer.py CHANGED
@@ -9,19 +9,17 @@ from src.display.css_html_js import style_content
9
  from src.envs import NUM_LINES_VISUALIZE
10
  from src.logging import log_file
11
 
12
- proj_dir = Path(__name__).parent
13
 
14
-
15
- def log_file_to_html_string():
16
  with open(log_file, "rt") as f:
17
- # Seek to the end of the file minus 300 lines
18
- # Read the last 300 lines of the file
19
- lines = f.readlines()
20
- lines = lines[-NUM_LINES_VISUALIZE:]
 
21
 
22
- # Syntax-highlight the last 300 lines of the file using the Python lexer and Monokai style
23
- output = "".join(reversed(lines))
24
- syntax = Syntax(output, "python", theme="monokai", word_wrap=True)
25
 
26
  console = Console(record=True, width=150, style="#272822", file=StringIO())
27
  console.print(syntax)
@@ -30,7 +28,7 @@ def log_file_to_html_string():
30
  # Parse the HTML content using BeautifulSoup
31
  soup = BeautifulSoup(html_content, 'lxml')
32
 
33
- # Modify the <pre> tag
34
  pre_tag = soup.pre
35
  pre_tag['class'] = 'scrollable'
36
  del pre_tag['style']
 
9
  from src.envs import NUM_LINES_VISUALIZE
10
  from src.logging import log_file
11
 
 
12
 
13
+ def log_file_to_html_string(reverse=True):
 
14
  with open(log_file, "rt") as f:
15
+ lines = f.readlines()
16
+ lines = lines[-NUM_LINES_VISUALIZE:]
17
+
18
+ if reverse:
19
+ lines = reversed(lines)
20
 
21
+ output = "".join(lines)
22
+ syntax = Syntax(output, "python", theme="monokai", word_wrap=True)
 
23
 
24
  console = Console(record=True, width=150, style="#272822", file=StringIO())
25
  console.print(syntax)
 
28
  # Parse the HTML content using BeautifulSoup
29
  soup = BeautifulSoup(html_content, 'lxml')
30
 
31
+ # Modify the <pre> tag and add custom styles
32
  pre_tag = soup.pre
33
  pre_tag['class'] = 'scrollable'
34
  del pre_tag['style']
src/logging.py CHANGED
@@ -23,32 +23,16 @@ def setup_logger(name: str):
23
 
24
  return logger
25
 
26
- # class Logger:
27
- # def __init__(self):
28
- # self.terminal = sys.stdout
29
- # self.log = open(log_file, "a+")
30
- #
31
- # def write(self, message):
32
- # self.terminal.write(message)
33
- # self.log.write(message)
34
- #
35
- # def flush(self):
36
- # self.terminal.flush()
37
- # self.log.flush()
38
- #
39
- # def isatty(self):
40
- # return False
41
- #
42
- # def read_logs():
43
- # sys.stdout.flush()
44
- # #API.upload_file(
45
- # # path_or_fileobj="output.log",
46
- # # path_in_repo="demo-backend.log",
47
- # # repo_id="demo-leaderboard-backend/logs",
48
- # # repo_type="dataset",
49
- # #)
50
- #
51
- # with open(log_file, "r") as f:
52
- # return f.read()
53
- #
54
- # LOGGER = Logger()
 
23
 
24
  return logger
25
 
26
+
27
+ def configure_root_logger():
28
+ # Configure the root logger
29
+ logging.basicConfig(level=logging.INFO)
30
+ root_logger = logging.getLogger()
31
+
32
+ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
33
+
34
+ file_handler = logging.FileHandler(log_file)
35
+ file_handler.setLevel(logging.INFO)
36
+ file_handler.setFormatter(formatter)
37
+
38
+ root_logger.addHandler(file_handler)