File size: 30,582 Bytes
4fc8c26 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 | {
"eval_loss": 1774.802567191746,
"eval_classification_loss": 0.0,
"eval_structure_loss": 399.6148203974185,
"eval_event_structure_loss": 1353.3075840162194,
"eval_count_loss": 21.880180027173914,
"step": 180,
"epoch": 4,
"eval_entity_strict_micro_precision": 0.5529135967849966,
"eval_entity_strict_micro_recall": 0.6277566539923954,
"eval_entity_strict_micro_f1": 0.5879629629629629,
"eval_entity_strict_macro_precision": 0.5591389029857231,
"eval_entity_strict_macro_recall": 0.6517493136533866,
"eval_entity_strict_macro_f1": 0.5972203337249327,
"eval_entity_strict_support": 2630,
"eval_entity_strict_classification_report": "label precision recall f1 support\n----------------------------------------------------------------------------------\nCVE 0.9429 0.9706 0.9565 34\nCapabilities 0.4422 0.3186 0.3704 204\nData 0.4602 0.5745 0.5110 141\nDevice 0.3600 0.5192 0.4252 52\nFile 0.6173 0.6250 0.6211 80\nGPE 0.4857 0.8500 0.6182 20\nMalware 0.4177 0.6000 0.4925 55\nMoney 0.8780 0.9231 0.9000 39\nNumber 0.7045 1.0000 0.8267 31\nOrganization 0.5524 0.6606 0.6017 439\nPII 0.6331 0.7213 0.6743 122\nPatch 0.6044 0.6790 0.6395 81\nPaymentMethod 0.3636 0.4211 0.3902 19\nPerson 0.5767 0.6681 0.6190 467\nPurpose 0.5333 0.5000 0.5161 64\nSoftware 0.5545 0.6289 0.5894 97\nSystem 0.4851 0.4545 0.4693 143\nTime 0.5732 0.8457 0.6833 162\nVersion 0.5224 0.5932 0.5556 59\nVulnerability 0.5774 0.6172 0.5967 290\nWebsite 0.4571 0.5161 0.4848 31\n----------------------------------------------------------------------------------\nmicro avg 0.5529 0.6278 0.5880 2630\nmacro avg 0.5591 0.6517 0.5972 2630",
"eval_entity_relaxed_micro_precision": 0.6684527796383122,
"eval_entity_relaxed_micro_recall": 0.75893536121673,
"eval_entity_relaxed_micro_f1": 0.7108262108262109,
"eval_entity_relaxed_macro_precision": 0.6809059873286564,
"eval_entity_relaxed_macro_recall": 0.7883380586752793,
"eval_entity_relaxed_macro_f1": 0.724903055770719,
"eval_entity_relaxed_support": 2630,
"eval_entity_relaxed_classification_report": "label precision recall f1 support\n----------------------------------------------------------------------------------\nCVE 0.9429 0.9706 0.9565 34\nCapabilities 0.7007 0.5049 0.5869 204\nData 0.6193 0.7730 0.6877 141\nDevice 0.5600 0.8077 0.6614 52\nFile 0.6914 0.7000 0.6957 80\nGPE 0.5143 0.9000 0.6545 20\nMalware 0.4937 0.7091 0.5821 55\nMoney 0.9024 0.9487 0.9250 39\nNumber 0.7045 1.0000 0.8267 31\nOrganization 0.6476 0.7745 0.7054 439\nPII 0.7122 0.8115 0.7586 122\nPatch 0.7473 0.8395 0.7907 81\nPaymentMethod 0.6818 0.7895 0.7317 19\nPerson 0.6599 0.7645 0.7083 467\nPurpose 0.6833 0.6406 0.6613 64\nSoftware 0.6091 0.6907 0.6473 97\nSystem 0.6343 0.5944 0.6137 143\nTime 0.6360 0.9383 0.7581 162\nVersion 0.7910 0.8983 0.8413 59\nVulnerability 0.7387 0.7897 0.7633 290\nWebsite 0.6286 0.7097 0.6667 31\n----------------------------------------------------------------------------------\nmicro avg 0.6685 0.7589 0.7108 2630\nmacro avg 0.6809 0.7883 0.7249 2630",
"eval_event_type_strict_micro_precision": 1.0,
"eval_event_type_strict_micro_recall": 0.7635135135135135,
"eval_event_type_strict_micro_f1": 0.8659003831417624,
"eval_event_type_strict_macro_precision": 1.0,
"eval_event_type_strict_macro_recall": 0.7539296446547968,
"eval_event_type_strict_macro_f1": 0.8470138684878789,
"eval_event_type_strict_support": 296,
"eval_event_type_strict_classification_report": "label precision recall f1 support\n----------------------------------------------------------------------------------\nCyber.Databreach 1.0000 0.7765 0.8742 85\nCyber.DiscoverVulnerability 1.0000 0.9138 0.9550 58\nCyber.PatchVulnerability 1.0000 0.9630 0.9811 54\nCyber.Phishing 1.0000 0.6458 0.7848 48\nCyber.Ransom 1.0000 0.4706 0.6400 51\n----------------------------------------------------------------------------------\nmicro avg 1.0000 0.7635 0.8659 296\nmacro avg 1.0000 0.7539 0.8470 296",
"eval_event_type_relaxed_micro_precision": 1.0,
"eval_event_type_relaxed_micro_recall": 0.7635135135135135,
"eval_event_type_relaxed_micro_f1": 0.8659003831417624,
"eval_event_type_relaxed_macro_precision": 1.0,
"eval_event_type_relaxed_macro_recall": 0.7539296446547968,
"eval_event_type_relaxed_macro_f1": 0.8470138684878789,
"eval_event_type_relaxed_support": 296,
"eval_event_type_relaxed_classification_report": "label precision recall f1 support\n----------------------------------------------------------------------------------\nCyber.Databreach 1.0000 0.7765 0.8742 85\nCyber.DiscoverVulnerability 1.0000 0.9138 0.9550 58\nCyber.PatchVulnerability 1.0000 0.9630 0.9811 54\nCyber.Phishing 1.0000 0.6458 0.7848 48\nCyber.Ransom 1.0000 0.4706 0.6400 51\n----------------------------------------------------------------------------------\nmicro avg 1.0000 0.7635 0.8659 296\nmacro avg 1.0000 0.7539 0.8470 296",
"eval_event_trigger_strict_micro_precision": 0.7432762836185819,
"eval_event_trigger_strict_micro_recall": 0.30218687872763417,
"eval_event_trigger_strict_micro_f1": 0.42968197879858655,
"eval_event_trigger_strict_macro_precision": 0.7657019360378305,
"eval_event_trigger_strict_macro_recall": 0.3092131289017904,
"eval_event_trigger_strict_macro_f1": 0.4180481150276771,
"eval_event_trigger_strict_support": 1006,
"eval_event_trigger_strict_classification_report": "label precision recall f1 support\n----------------------------------------------------------------------------------\nCyber.Databreach 0.7326 0.2614 0.3853 241\nCyber.DiscoverVulnerability 0.6691 0.4346 0.5269 214\nCyber.PatchVulnerability 0.8019 0.5060 0.6204 168\nCyber.Phishing 0.7917 0.2289 0.3551 166\nCyber.Ransom 0.8333 0.1152 0.2024 217\n----------------------------------------------------------------------------------\nmicro avg 0.7433 0.3022 0.4297 1006\nmacro avg 0.7657 0.3092 0.4180 1006",
"eval_event_trigger_relaxed_micro_precision": 0.8655256723716381,
"eval_event_trigger_relaxed_micro_recall": 0.3518886679920477,
"eval_event_trigger_relaxed_micro_f1": 0.5003533568904593,
"eval_event_trigger_relaxed_macro_precision": 0.897290194171999,
"eval_event_trigger_relaxed_macro_recall": 0.3603784452251194,
"eval_event_trigger_relaxed_macro_f1": 0.4881937078472333,
"eval_event_trigger_relaxed_support": 1006,
"eval_event_trigger_relaxed_classification_report": "label precision recall f1 support\n----------------------------------------------------------------------------------\nCyber.Databreach 0.8721 0.3112 0.4587 241\nCyber.DiscoverVulnerability 0.7554 0.4907 0.5949 214\nCyber.PatchVulnerability 0.9340 0.5893 0.7226 168\nCyber.Phishing 0.9583 0.2771 0.4299 166\nCyber.Ransom 0.9667 0.1336 0.2348 217\n----------------------------------------------------------------------------------\nmicro avg 0.8655 0.3519 0.5004 1006\nmacro avg 0.8973 0.3604 0.4882 1006",
"eval_event_argument_strict_micro_precision": 0.12080536912751678,
"eval_event_argument_strict_micro_recall": 0.04656319290465632,
"eval_event_argument_strict_micro_f1": 0.06721792477994132,
"eval_event_argument_strict_macro_precision": 0.13484134956588856,
"eval_event_argument_strict_macro_recall": 0.04791304452684179,
"eval_event_argument_strict_macro_f1": 0.06848885684243503,
"eval_event_argument_strict_support": 2706,
"eval_event_argument_strict_classification_report": "label precision recall f1 support\n----------------------------------------------------------------------------------\nAttack-Pattern 0.3846 0.0417 0.0752 120\nAttacker 0.1875 0.0545 0.0845 220\nCVE 0.0833 0.0435 0.0571 46\nCapabilities 0.0000 0.0000 0.0000 64\nCompromised-Data 0.1923 0.1008 0.1323 248\nDamage-Amount 0.0000 0.0000 0.0000 5\nDiscoverer 0.0649 0.0312 0.0422 160\nIssues-Addressed 0.0000 0.0000 0.0000 21\nNumber-of-Data 0.0000 0.0000 0.0000 13\nNumber-of-Victim 0.4286 0.1765 0.2500 17\nPatch 0.0164 0.0111 0.0132 90\nPatch-Number 0.1000 0.0500 0.0667 20\nPayment-Method 0.6000 0.1579 0.2500 19\nPlace 0.2222 0.1053 0.1429 19\nPrice 0.3077 0.1176 0.1702 34\nPurpose 0.0588 0.0156 0.0247 64\nReleaser 0.1231 0.0920 0.1053 87\nSupported_Platform 0.0000 0.0000 0.0000 8\nTime 0.1235 0.0588 0.0797 170\nTool 0.2258 0.0560 0.0897 125\nTrusted-Entity 0.1304 0.0405 0.0619 74\nVictim 0.1538 0.0467 0.0717 471\nVulnerability 0.0423 0.0190 0.0263 315\nVulnerable_System 0.0606 0.0269 0.0373 223\nVulnerable_System_Owner 0.0000 0.0000 0.0000 29\nVulnerable_System_Version 0.0000 0.0000 0.0000 44\n----------------------------------------------------------------------------------\nmicro avg 0.1208 0.0466 0.0672 2706\nmacro avg 0.1348 0.0479 0.0685 2706",
"eval_event_argument_relaxed_micro_precision": 0.8194980694980695,
"eval_event_argument_relaxed_micro_recall": 0.3401442307692308,
"eval_event_argument_relaxed_micro_f1": 0.4807474518686297,
"eval_event_argument_relaxed_macro_precision": 0.831740832726167,
"eval_event_argument_relaxed_macro_recall": 0.339162827753799,
"eval_event_argument_relaxed_macro_f1": 0.46016643844507155,
"eval_event_argument_relaxed_support": 2496,
"eval_event_argument_relaxed_classification_report": "label precision recall f1 support\n----------------------------------------------------------------------------------\nAttack-Pattern 0.7692 0.0855 0.1538 117\nAttacker 0.7344 0.2374 0.3588 198\nCVE 0.9583 0.5750 0.7188 40\nCapabilities 1.0000 0.0781 0.1449 64\nCompromised-Data 0.8682 0.4807 0.6188 233\nDamage-Amount 1.0000 0.6000 0.7500 5\nDiscoverer 0.8026 0.4326 0.5622 141\nIssues-Addressed 0.0000 0.0000 0.0000 21\nNumber-of-Data 1.0000 0.3077 0.4706 13\nNumber-of-Victim 1.0000 0.3529 0.5217 17\nPatch 0.8361 0.6623 0.7391 77\nPatch-Number 0.7000 0.3500 0.4667 20\nPayment-Method 1.0000 0.2632 0.4167 19\nPlace 1.0000 0.4211 0.5926 19\nPrice 1.0000 0.3529 0.5217 34\nPurpose 0.7647 0.2031 0.3210 64\nReleaser 0.8462 0.7857 0.8148 70\nSupported_Platform 0.6667 0.2500 0.3636 8\nTime 0.9383 0.4691 0.6255 162\nTool 0.7742 0.2087 0.3288 115\nTrusted-Entity 0.8261 0.2794 0.4176 68\nVictim 0.7676 0.2460 0.3726 443\nVulnerability 0.8085 0.4014 0.5365 284\nVulnerable_System 0.7071 0.3553 0.4730 197\nVulnerable_System_Owner 0.8571 0.2609 0.4000 23\nVulnerable_System_Version 1.0000 0.1591 0.2745 44\n----------------------------------------------------------------------------------\nmicro avg 0.8195 0.3401 0.4807 2496\nmacro avg 0.8317 0.3392 0.4602 2496",
"eval_event_strict_micro_precision": 0.3909415971394517,
"eval_event_strict_micro_recall": 0.16367265469061876,
"eval_event_strict_micro_f1": 0.23074217376011252,
"eval_event_strict_macro_precision": 0.34262179913617374,
"eval_event_strict_macro_recall": 0.18226258404113393,
"eval_event_strict_macro_f1": 0.22516722765225256,
"eval_event_strict_support": 4008,
"eval_event_strict_classification_report": "label precision recall f1 support\n----------------------------------------------------------------------------------\narg:Attack-Pattern 0.3846 0.0417 0.0752 120\narg:Attacker 0.1875 0.0545 0.0845 220\narg:CVE 0.0833 0.0435 0.0571 46\narg:Capabilities 0.0000 0.0000 0.0000 64\narg:Compromised-Data 0.1923 0.1008 0.1323 248\narg:Damage-Amount 0.0000 0.0000 0.0000 5\narg:Discoverer 0.0649 0.0312 0.0422 160\narg:Issues-Addressed 0.0000 0.0000 0.0000 21\narg:Number-of-Data 0.0000 0.0000 0.0000 13\narg:Number-of-Victim 0.4286 0.1765 0.2500 17\narg:Patch 0.0164 0.0111 0.0132 90\narg:Patch-Number 0.1000 0.0500 0.0667 20\narg:Payment-Method 0.6000 0.1579 0.2500 19\narg:Place 0.2222 0.1053 0.1429 19\narg:Price 0.3077 0.1176 0.1702 34\narg:Purpose 0.0588 0.0156 0.0247 64\narg:Releaser 0.1231 0.0920 0.1053 87\narg:Supported_Platform 0.0000 0.0000 0.0000 8\narg:Time 0.1235 0.0588 0.0797 170\narg:Tool 0.2258 0.0560 0.0897 125\narg:Trusted-Entity 0.1304 0.0405 0.0619 74\narg:Victim 0.1538 0.0467 0.0717 471\narg:Vulnerability 0.0423 0.0190 0.0263 315\narg:Vulnerable_System 0.0606 0.0269 0.0373 223\narg:Vulnerable_System_Owner 0.0000 0.0000 0.0000 29\narg:Vulnerable_System_Version 0.0000 0.0000 0.0000 44\ntrigger:Cyber.Databreach 0.7326 0.2614 0.3853 241\ntrigger:Cyber.DiscoverVulnerability 0.6691 0.4346 0.5269 214\ntrigger:Cyber.PatchVulnerability 0.8019 0.5060 0.6204 168\ntrigger:Cyber.Phishing 0.7917 0.2289 0.3551 166\ntrigger:Cyber.Ransom 0.8333 0.1152 0.2024 217\ntype:Cyber.Databreach 1.0000 0.7765 0.8742 85\ntype:Cyber.DiscoverVulnerability 1.0000 0.9138 0.9550 58\ntype:Cyber.PatchVulnerability 1.0000 0.9630 0.9811 54\ntype:Cyber.Phishing 1.0000 0.6458 0.7848 48\ntype:Cyber.Ransom 1.0000 0.4706 0.6400 51\n----------------------------------------------------------------------------------\nmicro avg 0.3909 0.1637 0.2307 4008\nmacro avg 0.3426 0.1823 0.2252 4008",
"eval_event_relaxed_micro_precision": 0.8551765409934171,
"eval_event_relaxed_micro_recall": 0.3762506582411796,
"eval_event_relaxed_micro_f1": 0.5225818248308649,
"eval_event_relaxed_macro_precision": 0.864214239492787,
"eval_event_relaxed_macro_recall": 0.3997159436388433,
"eval_event_relaxed_macro_f1": 0.5177879244790952,
"eval_event_relaxed_support": 3798,
"eval_event_relaxed_classification_report": "label precision recall f1 support\n----------------------------------------------------------------------------------\narg:Attack-Pattern 0.7692 0.0855 0.1538 117\narg:Attacker 0.7344 0.2374 0.3588 198\narg:CVE 0.9583 0.5750 0.7188 40\narg:Capabilities 1.0000 0.0781 0.1449 64\narg:Compromised-Data 0.8682 0.4807 0.6188 233\narg:Damage-Amount 1.0000 0.6000 0.7500 5\narg:Discoverer 0.8026 0.4326 0.5622 141\narg:Issues-Addressed 0.0000 0.0000 0.0000 21\narg:Number-of-Data 1.0000 0.3077 0.4706 13\narg:Number-of-Victim 1.0000 0.3529 0.5217 17\narg:Patch 0.8361 0.6623 0.7391 77\narg:Patch-Number 0.7000 0.3500 0.4667 20\narg:Payment-Method 1.0000 0.2632 0.4167 19\narg:Place 1.0000 0.4211 0.5926 19\narg:Price 1.0000 0.3529 0.5217 34\narg:Purpose 0.7647 0.2031 0.3210 64\narg:Releaser 0.8462 0.7857 0.8148 70\narg:Supported_Platform 0.6667 0.2500 0.3636 8\narg:Time 0.9383 0.4691 0.6255 162\narg:Tool 0.7742 0.2087 0.3288 115\narg:Trusted-Entity 0.8261 0.2794 0.4176 68\narg:Victim 0.7676 0.2460 0.3726 443\narg:Vulnerability 0.8085 0.4014 0.5365 284\narg:Vulnerable_System 0.7071 0.3553 0.4730 197\narg:Vulnerable_System_Owner 0.8571 0.2609 0.4000 23\narg:Vulnerable_System_Version 1.0000 0.1591 0.2745 44\ntrigger:Cyber.Databreach 0.8721 0.3112 0.4587 241\ntrigger:Cyber.DiscoverVulnerability 0.7554 0.4907 0.5949 214\ntrigger:Cyber.PatchVulnerability 0.9340 0.5893 0.7226 168\ntrigger:Cyber.Phishing 0.9583 0.2771 0.4299 166\ntrigger:Cyber.Ransom 0.9667 0.1336 0.2348 217\ntype:Cyber.Databreach 1.0000 0.7765 0.8742 85\ntype:Cyber.DiscoverVulnerability 1.0000 0.9138 0.9550 58\ntype:Cyber.PatchVulnerability 1.0000 0.9630 0.9811 54\ntype:Cyber.Phishing 1.0000 0.6458 0.7848 48\ntype:Cyber.Ransom 1.0000 0.4706 0.6400 51\n----------------------------------------------------------------------------------\nmicro avg 0.8552 0.3763 0.5226 3798\nmacro avg 0.8642 0.3997 0.5178 3798",
"eval_entity_error_COR": 1679,
"eval_entity_error_LE": 51,
"eval_entity_error_BES": 147,
"eval_entity_error_BEL": 127,
"eval_entity_error_BEO": 40,
"eval_entity_error_LBE": 84,
"eval_entity_error_FP": 858,
"eval_entity_error_FN": 502,
"eval_entity_fair_micro_precision": 0.608002896976281,
"eval_entity_fair_micro_recall": 0.6979837871544378,
"eval_entity_fair_micro_f1": 0.6498935552544998,
"eval_entity_fair_support": 2630,
"eval_entity_error_report": "entity error analysis (Ortmann 2022, fair evaluation)\ntype count\n-----------------\nCOR 1679\nLE 51\nBES 147\nBEL 127\nBEO 40\nLBE 84\nFP 858\nFN 502\n-----------------\nfair P / R / F1: 0.6080 / 0.6980 / 0.6499\n\nlabel confusions (gold -> pred):\n Person -> Organization: 13\n Organization -> Person: 8\n Capabilities -> Person: 7\n Capabilities -> Malware: 5\n Capabilities -> Organization: 5\n PII -> Data: 5\n PII -> Person: 4\n System -> Organization: 4\n Vulnerability -> System: 4\n Capabilities -> File: 3\n Data -> File: 3\n Purpose -> Data: 3\n Capabilities -> Data: 2\n Capabilities -> Purpose: 2\n Capabilities -> System: 2\n Capabilities -> Vulnerability: 2\n Data -> Capabilities: 2\n Data -> PII: 2\n File -> Organization: 2\n Organization -> Time: 2\n Patch -> Software: 2\n Patch -> Vulnerability: 2\n Person -> PII: 2\n Purpose -> Organization: 2\n Software -> Patch: 2\n System -> Device: 2\n System -> Software: 2\n Vulnerability -> Person: 2\n Vulnerability -> Software: 2\n Capabilities -> GPE: 1\n Capabilities -> PII: 1\n Capabilities -> Software: 1\n Capabilities -> Website: 1\n Data -> System: 1\n Device -> Organization: 1\n Device -> Software: 1\n File -> Malware: 1\n File -> PII: 1\n Malware -> Capabilities: 1\n Money -> PaymentMethod: 1\n Organization -> Device: 1\n Organization -> GPE: 1\n Organization -> Software: 1\n Organization -> Website: 1\n PII -> Capabilities: 1\n PII -> Purpose: 1\n Patch -> Organization: 1\n PaymentMethod -> Time: 1\n Person -> Software: 1\n Person -> Version: 1\n Person -> Vulnerability: 1\n Purpose -> PII: 1\n Purpose -> Time: 1\n Purpose -> Vulnerability: 1\n Software -> Capabilities: 1\n Software -> Version: 1\n System -> Capabilities: 1\n System -> Data: 1\n System -> Vulnerability: 1\n Version -> Vulnerability: 1\n Vulnerability -> Capabilities: 1\n Vulnerability -> Organization: 1\n Vulnerability -> Time: 1\n Website -> File: 1\n Website -> Organization: 1\n Website -> System: 1",
"eval_event_trigger_error_COR": 304,
"eval_event_trigger_error_LE": 1,
"eval_event_trigger_error_BES": 34,
"eval_event_trigger_error_BEL": 11,
"eval_event_trigger_error_BEO": 5,
"eval_event_trigger_error_LBE": 0,
"eval_event_trigger_error_FP": 54,
"eval_event_trigger_error_FN": 651,
"eval_event_trigger_fair_micro_precision": 0.7926988265971316,
"eval_event_trigger_fair_micro_recall": 0.3100458949515553,
"eval_event_trigger_fair_micro_f1": 0.44574780058651026,
"eval_event_trigger_fair_support": 1006,
"eval_event_trigger_error_report": "event_trigger error analysis (Ortmann 2022, fair evaluation)\ntype count\n-----------------\nCOR 304\nLE 1\nBES 34\nBEL 11\nBEO 5\nLBE 0\nFP 54\nFN 651\n-----------------\nfair P / R / F1: 0.7927 / 0.3100 / 0.4457\n\nlabel confusions (gold -> pred):\n Cyber.Databreach -> Cyber.Ransom: 1",
"eval_event_argument_error_COR": 751,
"eval_event_argument_error_LE": 23,
"eval_event_argument_error_BES": 32,
"eval_event_argument_error_BEL": 43,
"eval_event_argument_error_BEO": 14,
"eval_event_argument_error_LBE": 24,
"eval_event_argument_error_FP": 103,
"eval_event_argument_error_FN": 1556,
"eval_event_argument_fair_micro_precision": 0.8145336225596529,
"eval_event_argument_fair_micro_recall": 0.3162105263157895,
"eval_event_argument_fair_micro_f1": 0.4555656657567485,
"eval_event_argument_fair_support": 2443,
"eval_event_argument_error_report": "event_argument error analysis (Ortmann 2022, fair evaluation)\ntype count\n-----------------\nCOR 751\nLE 23\nBES 32\nBEL 43\nBEO 14\nLBE 24\nFP 103\nFN 1556\n-----------------\nfair P / R / F1: 0.8145 / 0.3162 / 0.4556\n\nlabel confusions (gold -> pred):\n Vulnerability -> Vulnerable_System: 5\n Attacker -> Victim: 3\n Purpose -> Compromised-Data: 3\n Attack-Pattern -> Tool: 2\n Attack-Pattern -> Trusted-Entity: 2\n Discoverer -> Attacker: 2\n Discoverer -> Releaser: 2\n Releaser -> Discoverer: 2\n Trusted-Entity -> Victim: 2\n Victim -> Trusted-Entity: 2\n Vulnerability -> Releaser: 2\n Vulnerable_System_Owner -> Vulnerable_System: 2\n Attack-Pattern -> Purpose: 1\n Attack-Pattern -> Releaser: 1\n Capabilities -> Compromised-Data: 1\n Compromised-Data -> Victim: 1\n Issues-Addressed -> Releaser: 1\n Patch-Number -> Vulnerable_System: 1\n Tool -> Vulnerability: 1\n Victim -> Attack-Pattern: 1\n Victim -> Releaser: 1\n Victim -> Vulnerable_System: 1\n Victim -> Vulnerable_System_Owner: 1\n Vulnerability -> Discoverer: 1\n Vulnerable_System -> Discoverer: 1\n Vulnerable_System -> Releaser: 1\n Vulnerable_System -> Supported_Platform: 1\n Vulnerable_System -> Vulnerability: 1\n Vulnerable_System_Owner -> Vulnerability: 1\n Vulnerable_System_Version -> Patch-Number: 1"
} |