1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
|
#!/usr/bin/python3
#
# Copyright Linux Foundation, Richard Purdie
#
# SPDX-License-Identifier: GPL-2.0-only
#
import signal
import os
import sys
import threading
import time
tmpfile = '/tmp/.buildworker-janitor'+os.getcwd().replace('/', '-')
sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'scripts'))
import utils
ourconfig = utils.loadconfig()
trashdir = utils.getconfig("TRASH_DIR", ourconfig)
mirrordir = utils.getconfig("REPO_STASH_DIR", ourconfig)
if not trashdir:
print("Please set TRASH_DIR in the configuration file")
sys.exit(1)
if not mirrordir:
print("Please set REPO_STASH_DIR in the configuration file")
sys.exit(1)
def trash_processor(trashdir):
print("Monitoring trashdir %s" % trashdir)
if not os.path.exists(trashdir):
os.makedirs(trashdir)
if trashdir == "/":
print("Not prepared to use a trashdir of /")
return
while True:
try:
files = os.listdir(trashdir)
if files:
for file in files:
file_path = trashdir + "/" + file
file_age = time.time() - os.path.getmtime(file_path)
if file_age >= 60:
os.system("nice -n 10 ionice -c 3 rm %s -rf" % file_path)
else:
print("Not removing '%s' - age is only %s seconds. There may be another process using it" % (file_path, str(int(file_age))))
else:
time.sleep(120*60) # 30 minutes
except Exception as e:
print("Exception %s in trash cleaner" % str(e))
time.sleep(60) # 1 minute timeout to prevent crazy looping
pass
return
def mirror_processor(mirrordir):
print("Updating mirrors in %s" % mirrordir)
mirrorpaths = []
for repo in ourconfig["repo-defaults"]:
mirror = ourconfig["repo-defaults"][repo]["url"]
mirrorpath = os.path.join(mirrordir, repo)
mirrorpaths.append(mirrorpath)
if not os.path.exists(mirrorpaths[-1] + "/.git/"):
os.system("git clone --bare --mirror %s %s" % (mirror, mirrorpaths[-1]))
while True:
for path in mirrorpaths:
os.chdir(path)
os.system("git fetch --prune --all")
time.sleep(30*60) # 30 minutes
return
#Check to see if this is running already. If so, kill it and rerun
if os.path.exists(tmpfile) and os.path.isfile(tmpfile):
print("A prior PID file exists. Attempting to kill.")
with open(tmpfile, 'r') as f:
pid=f.readline()
try:
os.kill(int(pid), signal.SIGKILL)
# We need to sleep for a second or two just to give the SIGKILL time
time.sleep(2)
except OSError as ex:
print("""We weren't able to kill the prior buildworker-janitor. Trying again.""")
pass
# Check if the process that we killed is alive.
try:
os.kill(int(pid), 0)
except OSError as ex:
pass
elif os.path.exists(tmpfile) and not os.path.isfile(tmpfile):
raise Exception("""/tmp/.buildworker-janitor is a director. remove it to continue.""")
try:
os.unlink(tmpfile)
except:
pass
with open(tmpfile, 'w') as f:
print(os.getpid(), file=f)
threads = []
threads.append(threading.Thread(target=trash_processor, args=(trashdir,)))
threads[-1].start()
threads.append(threading.Thread(target=mirror_processor, args=(mirrordir,)))
threads[-1].start()
# wait for all threads to finish
for t in threads:
t.join()
sys.exit(0)
|
