adding faiss similarity search tests

CPU install routine
sortimg
2025-05-08 08:31:28 +00:00 · 2024-08-11 11:21:20 +02:00 · 2024-08-11 08:58:20 +02:00 · 2024-08-11 08:42:41 +02:00 · 2024-08-11 08:41:35 +02:00 · 2024-08-10 17:39:43 +02:00
32 changed files with 61113 additions and 0 deletions
--- a/2-3-2-open-source-deep-learning-models-for-encoding-taks/Comparison_of_Encoder_Models.ipynb
+++ b/2-3-2-open-source-deep-learning-models-for-encoding-taks/Comparison_of_Encoder_Models.ipynb
--- a/2-3-2-open-source-deep-learning-models-for-encoding-taks/Comparison_of_Encoder_Models_LongFormer_and_Linformer.ipynb
+++ b/2-3-2-open-source-deep-learning-models-for-encoding-taks/Comparison_of_Encoder_Models_LongFormer_and_Linformer.ipynb
--- a/2-5-automated-machine-learning-with-gp/TPOT_Foundations.ipynb
+++ b/2-5-automated-machine-learning-with-gp/TPOT_Foundations.ipynb
--- a/2-5-automated-machine-learning-with-gp/corpus_foundations_thesis.zip
+++ b/2-5-automated-machine-learning-with-gp/corpus_foundations_thesis.zip
--- a/2-5-automated-machine-learning-with-gp/new_workbook_13.xlsm
+++ b/2-5-automated-machine-learning-with-gp/new_workbook_13.xlsm
--- a/2-5-automated-machine-learning-with-gp/new_workbook_13_EvilClippy.xlsm
+++ b/2-5-automated-machine-learning-with-gp/new_workbook_13_EvilClippy.xlsm
--- a/2-6-1-elasticsearch-api-access-polars/Elasticsearch.ipynb
+++ b/2-6-1-elasticsearch-api-access-polars/Elasticsearch.ipynb
--- a/3-1-adversary-emulation-and-training-data-generation/Sysmon_Observations
+++ b/3-1-adversary-emulation-and-training-data-generation/Sysmon_Observations
--- a/3-1-adversary-emulation-and-training-data-generation/data_sources_new_3.json
+++ b/3-1-adversary-emulation-and-training-data-generation/data_sources_new_3.json
--- a/3-1-adversary-emulation-and-training-data-generation/sysmon_layer.yaml
+++ b/3-1-adversary-emulation-and-training-data-generation/sysmon_layer.yaml
--- a/3-1-adversary-emulation-and-training-data-generation/sysmon_observations.xlsx
+++ b/3-1-adversary-emulation-and-training-data-generation/sysmon_observations.xlsx
--- a/3-1-adversary-emulation-and-training-data-generation/sysmonconfig-export.xml
+++ b/3-1-adversary-emulation-and-training-data-generation/sysmonconfig-export.xml
--- a/3-2-test-corpus-initial-access-mw-and-3-3-simulator/Book1_Foundations.xlsm
+++ b/3-2-test-corpus-initial-access-mw-and-3-3-simulator/Book1_Foundations.xlsm
--- a/3-2-test-corpus-initial-access-mw-and-3-3-simulator/GruntHTTP-vba-stager.txt
+++ b/3-2-test-corpus-initial-access-mw-and-3-3-simulator/GruntHTTP-vba-stager.txt
--- a/3-2-test-corpus-initial-access-mw-and-3-3-simulator/GruntHTTP.exe
+++ b/3-2-test-corpus-initial-access-mw-and-3-3-simulator/GruntHTTP.exe
--- a/3-2-test-corpus-initial-access-mw-and-3-3-simulator/Readme.md
+++ b/3-2-test-corpus-initial-access-mw-and-3-3-simulator/Readme.md
--- a/3-2-test-corpus-initial-access-mw-and-3-3-simulator/Simulator/enable_content.png
+++ b/3-2-test-corpus-initial-access-mw-and-3-3-simulator/Simulator/enable_content.png
--- a/3-2-test-corpus-initial-access-mw-and-3-3-simulator/Simulator/requirements.txt
+++ b/3-2-test-corpus-initial-access-mw-and-3-3-simulator/Simulator/requirements.txt
--- a/3-2-test-corpus-initial-access-mw-and-3-3-simulator/Simulator/simulator.py
+++ b/3-2-test-corpus-initial-access-mw-and-3-3-simulator/Simulator/simulator.py
--- a/3-2-test-corpus-initial-access-mw-and-3-3-simulator/corpus_builder/agentic_corpus_generator.ps1
+++ b/3-2-test-corpus-initial-access-mw-and-3-3-simulator/corpus_builder/agentic_corpus_generator.ps1
--- a/3-2-test-corpus-initial-access-mw-and-3-3-simulator/corpus_builder/create_macro_xls.ps1
+++ b/3-2-test-corpus-initial-access-mw-and-3-3-simulator/corpus_builder/create_macro_xls.ps1
--- a/3-2-test-corpus-initial-access-mw-and-3-3-simulator/corpus_builder/macro.vbs
+++ b/3-2-test-corpus-initial-access-mw-and-3-3-simulator/corpus_builder/macro.vbs
--- a/3-2-test-corpus-initial-access-mw-and-3-3-simulator/corpus_builder/new_workbook_3.xlsm
+++ b/3-2-test-corpus-initial-access-mw-and-3-3-simulator/corpus_builder/new_workbook_3.xlsm
--- a/3-2-test-corpus-initial-access-mw-and-3-3-simulator/corpus_builder/template.xlsx
+++ b/3-2-test-corpus-initial-access-mw-and-3-3-simulator/corpus_builder/template.xlsx
--- a/3-2-test-corpus-initial-access-mw-and-3-3-simulator/file.exe
+++ b/3-2-test-corpus-initial-access-mw-and-3-3-simulator/file.exe
--- a/3-7-security-and-performance-log-analysis-sw/Elasticsearch-Pandas-vs-Polars-May-15-2024.ipynb
+++ b/3-7-security-and-performance-log-analysis-sw/Elasticsearch-Pandas-vs-Polars-May-15-2024.ipynb
--- a/4-5-attention-based-log-similarity-clustering-with-faiss/Log-Similarity-Clustering-FAISS.ipynb
+++ b/4-5-attention-based-log-similarity-clustering-with-faiss/Log-Similarity-Clustering-FAISS.ipynb
--- a/4-5-attention-based-log-similarity-clustering-with-faiss/log_tokenizer.json
+++ b/4-5-attention-based-log-similarity-clustering-with-faiss/log_tokenizer.json
--- a/README.md
+++ b/README.md
@ -0,0 +1,5 @@
+[![DOI](https://zenodo.org/badge/744004649.svg)](https://zenodo.org/doi/10.5281/zenodo.13208293)
+
+
+# Master Thesis: Development and Evaluation of Software for Forensic Log-Analysis Using Machine Learning and Genetic Programming
+
--- a/dependencies/install_cpu.sh
+++ b/dependencies/install_cpu.sh
@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+if [[ "$(uname)" == "Darwin" ]] || [[ "$(uname)" == "Linux" ]]; then
+  echo "The CPU installation is starting (macOS, Linux)."
+
+  echo "Installing dependencies"
+  pip install -r "https://raw.githubusercontent.com/norandom/log2ml/main/dependencies/requirements.cgpu.txt"
+
+  echo "cuML will not be installed."
+
+else
+  echo "This does not appear to be a CPU environment."
+fi
--- a/dependencies/requirements.cpu.txt
+++ b/dependencies/requirements.cpu.txt
@ -0,0 +1,29 @@
+openai==1.13.3
+langchain==0.1.10
+python-dotenv==1.0.1
+tiktoken==0.6.0
+langchain_openai==0.0.8
+langchain_experimental==0.0.53
+langchainhub==0.1.14
+ipywidgets
+transformers==4.39.0
+torch==2.2.1+cpu
+torchvision==0.17.1+cpu
+torchaudio==2.2.1+cpu
+sentence-transformers==2.5.1
+faiss_cpu==1.8.0
+linformer-pytorch==0.19.3
+PyGithub==2.3.0
+deap==1.4.1
+update_checker==0.18.0
+scikit-mdr==0.4.5
+skrebate==0.62
+xgboost==2.0.3
+stopit==1.1.2
+tpot==0.12.2
+umap-learn==0.5.6
+tabulate==0.9.0
+onnx==1.16.1
+skl2onnx==1.17.0
+protobuf==3.20.3
+dill==0.3.8
--- a/workbench/LinFormer_AutoML_on_AE_sysmon_dataset_(Excel_implant_C2).ipynb
+++ b/workbench/LinFormer_AutoML_on_AE_sysmon_dataset_(Excel_implant_C2).ipynb
Author	SHA1	Message	Date
marius	c09b7eec14	adding faiss similarity search tests	2024-08-11 11:21:20 +02:00
marius	9f7f0028e3	CPU install routine	2024-08-11 08:58:20 +02:00
marius	432fb22b6d	sortimg	2024-08-11 08:42:41 +02:00
marius	c5addfd9a1	sorting	2024-08-11 08:41:35 +02:00
marius	9b5858772b	sorting	2024-08-10 17:39:43 +02:00
marius	eb62dace9f	sorting	2024-08-10 15:58:56 +02:00
marius	de9f9a5a0e	sorting	2024-08-10 12:37:28 +02:00
marius	4c0645e20b	sorting	2024-08-10 11:57:53 +02:00
Marius Ciepluch	001c431210	Create README.md	2024-08-04 09:24:14 +02:00