1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
|
#!/usr/bin/env python3
# SPDX-License-Identifier: CC0-1.0
#
# Copyright (C) 2024 Wojtek Kosior <koszko@koszko.org>
from dataclasses import dataclass
from html.parser import HTMLParser
from pathlib import Path
import sys
import requests
import yaml
mitre_pages_path = Path(".") / "scraping" / "attack.mitre.org"
profiles_path = Path('./profiles.yaml')
def mitre_page_download(path):
response = requests.get('https://attack.mitre.org/' + path)
response.raise_for_status()
return response.text
def mitre_page_get(path):
page_path = mitre_pages_path / path
if page_path.exists():
return page_path.read_text()
else:
if not page_path.parent.exists():
page_path.parent.mkdir(parents=True)
page_text = mitre_page_download(path)
page_path.write_text(page_text)
return page_text
@dataclass
class Group:
name: str
mitre_id: str
aliases: list[str]
class GroupListPageParser(HTMLParser):
def __init__(self, relevant_groups):
super().__init__()
self.relevant_groups = relevant_groups
self.col_numbers = [-1]
self.current_tags = ["*TOP*"]
self.collected_groups = {}
self.collecting_new_group()
def collecting_new_group(self):
self.current_group_mitre_id = None
self.current_group_name = None
self.current_group_aliases = None
def handle_starttag(self, tag, attrs):
self.current_tags.append(tag)
if tag == "tr":
self.col_numbers.append(-1)
elif tag == "td":
self.col_numbers[-1] += 1
def handle_data(self, data):
if self.current_tags[-1] == "a" and self.col_numbers[-1] == 0:
self.current_group_mitre_id = data.strip()
elif self.current_tags[-1] == "a" and self.col_numbers[-1] == 1:
self.current_group_name = data.strip()
elif self.current_tags[-1] == "td" and self.col_numbers[-1] == 2:
data = data.strip()
if data:
self.current_group_aliases = data.split(", ")
else:
self.current_group_aliases = []
def handle_endtag(self, tag):
self.current_tags.pop()
if tag == "tr":
self.col_numbers.pop()
if self.current_group_name is None or \
self.current_group_mitre_id is None or \
self.current_group_aliases is None:
print("Incomplete data for group.", file=sys.stderr)
return
if self.current_group_name not in self.relevant_groups:
print(f"Ignoring group `{self.current_group_name}'",
file=sys.stderr)
return
if self.current_group_name in self.collected_groups:
print(f"Double definition of group `{self.current_group_name}'",
file=sys.stderr)
return
self.collected_groups[self.current_group_name] = Group(
self.current_group_name,
self.current_group_mitre_id,
self.current_group_aliases
)
self.collecting_new_group()
def get_groups(names):
parser = GroupListPageParser(names)
parser.feed(mitre_page_get("groups/"))
return parser.collected_groups
def get_group_names(profiles_path):
def group_names(inp):
return {group["name"] for group in yaml.safe_load(inp)["groups"]}
if profiles_path:
with open(profiles_path) as inp:
return group_names(inp)
return group_names(sys.stdin)
if __name__ == "__main__":
group_names = get_group_names(None if len(sys.argv) < 2 else sys.argv[1])
groups = get_groups(group_names)
yaml.safe_dump([group.__dict__ for group in groups.values()], sys.stdout)
|
