|
1
|
# Extraction Scripts Reference |
|
2
|
|
|
3
|
Complete Python scripts for Phase 1 reconnaissance. Run these in order. |
|
4
|
|
|
5
|
## Setup |
|
6
|
|
|
7
|
```python |
|
8
|
|
|
9
|
from playwright.sync_api import sync_playwright |
|
10
|
|
|
11
|
import json, os, re, hashlib, urllib.request, urllib.error, subprocess |
|
12
|
|
|
13
|
CHROMIUM_PATH = "FILL_IN" \# from prerequisite step |
|
14
|
|
|
15
|
TARGET_URL = "FILL_IN" |
|
16
|
|
|
17
|
OUT_DIR = "clone-data" |
|
18
|
|
|
19
|
os.makedirs(f"{OUT_DIR}/screenshots", exist_ok=True) |
|
20
|
|
|
21
|
os.makedirs(f"{OUT_DIR}/components", exist_ok=True) |
|
22
|
|
|
23
|
``` |
|
24
|
|
|
25
|
## 1. Navigate and Save Raw HTML |
|
26
|
|
|
27
|
```python |
|
28
|
|
|
29
|
with sync_playwright() as p: |
|
30
|
|
|
31
|
browser = p.chromium.launch( |
|
32
|
|
|
33
|
headless=True, |
|
34
|
|
|
35
|
executable_path=CHROMIUM_PATH, |
|
36
|
|
|
37
|
args=["--no-sandbox", "--disable-setuid-sandbox"] |
|
38
|
|
|
39
|
) |
|
40
|
|
|
41
|
page = browser.new_page(viewport={"width": 1440, "height": 900}) |
|
42
|
|
|
43
|
page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=60000) |
|
44
|
|
|
45
|
page.wait_for_timeout(5000) |
|
46
|
|
|
47
|
# Scroll full page to trigger lazy loading |
|
48
|
|
|
49
|
for _ in range(20): |
|
50
|
|
|
51
|
page.evaluate("window.scrollBy(0, 500)") |
|
52
|
|
|
53
|
page.wait_for_timeout(500) |
|
54
|
|
|
55
|
page.evaluate("window.scrollTo(0, 0)") |
|
56
|
|
|
57
|
page.wait_for_timeout(2000) |
|
58
|
|
|
59
|
# CRITICAL: Save raw HTML as the source of truth |
|
60
|
|
|
61
|
raw_html = page.content() |
|
62
|
|
|
63
|
with open(f"{OUT_DIR}/raw.html", "w", encoding="utf-8") as f: |
|
64
|
|
|
65
|
f.write(raw_html) |
|
66
|
|
|
67
|
print(f"Saved raw.html ({len(raw_html)} chars)") |
|
68
|
|
|
69
|
``` |
|
70
|
|
|
71
|
## 2. Full-Page Screenshots |
|
72
|
|
|
73
|
```python |
|
74
|
|
|
75
|
# Desktop |
|
76
|
|
|
77
|
page.screenshot(path=f"{OUT_DIR}/screenshots/desktop-full.png", full_page=True) |
|
78
|
|
|
79
|
# Tablet |
|
80
|
|
|
81
|
page.set_viewport_size({"width": 768, "height": 1024}) |
|
82
|
|
|
83
|
page.wait_for_timeout(1000) |
|
84
|
|
|
85
|
page.screenshot(path=f"{OUT_DIR}/screenshots/tablet-full.png", full_page=True) |
|
86
|
|
|
87
|
# Mobile |
|
88
|
|
|
89
|
page.set_viewport_size({"width": 390, "height": 844}) |
|
90
|
|
|
91
|
page.wait_for_timeout(1000) |
|
92
|
|
|
93
|
page.screenshot(path=f"{OUT_DIR}/screenshots/mobile-full.png", full_page=True) |
|
94
|
|
|
95
|
# Reset to desktop |
|
96
|
|
|
97
|
page.set_viewport_size({"width": 1440, "height": 900}) |
|
98
|
|
|
99
|
page.wait_for_timeout(500) |
|
100
|
|
|
101
|
``` |
|
102
|
|
|
103
|
## 3. Per-Section Screenshots |
|
104
|
|
|
105
|
```python |
|
106
|
|
|
107
|
sections = page.query_selector_all('section, [class*="section"], main > div, main > section') |
|
108
|
|
|
109
|
for i, section in enumerate(sections): |
|
110
|
|
|
111
|
try: |
|
112
|
|
|
113
|
section.scroll_into_view_if_needed() |
|
114
|
|
|
115
|
page.wait_for_timeout(300) |
|
116
|
|
|
117
|
section.screenshot(path=f"{OUT_DIR}/screenshots/section-{i:02d}.png") |
|
118
|
|
|
119
|
print(f"Screenshot: section-{i:02d}") |
|
120
|
|
|
121
|
except Exception as e: |
|
122
|
|
|
123
|
print(f"Skip section {i}: {e}") |
|
124
|
|
|
125
|
``` |
|
126
|
|
|
127
|
## 4. Section Inventory Builder |
|
128
|
|
|
129
|
Run this after saving raw.html. It produces a markdown checklist. |
|
130
|
|
|
131
|
```python |
|
132
|
|
|
133
|
inventory = page.evaluate(""" |
|
134
|
|
|
135
|
() => { |
|
136
|
|
|
137
|
const main = document.querySelector('main') || document.body; |
|
138
|
|
|
139
|
const allSections = []; |
|
140
|
|
|
141
|
// Header |
|
142
|
|
|
143
|
const header = document.querySelector('header'); |
|
144
|
|
|
145
|
if (header) { |
|
146
|
|
|
147
|
allSections.push({ |
|
148
|
|
|
149
|
index: 0, |
|
150
|
|
|
151
|
selector: 'header', |
|
152
|
|
|
153
|
type: 'header', |
|
154
|
|
|
155
|
tag: header.tagName.toLowerCase(), |
|
156
|
|
|
157
|
classes: header.className?.toString().slice(0, 200) || '', |
|
158
|
|
|
159
|
headings: [], |
|
160
|
|
|
161
|
buttons: [...header.querySelectorAll('button, a[class*="btn"]')] |
|
162
|
|
|
163
|
.map(b => b.innerText.trim()).filter(t => t).slice(0, 5), |
|
164
|
|
|
165
|
imageCount: header.querySelectorAll('img').length, |
|
166
|
|
|
167
|
textPreview: header.innerText.trim().slice(0, 200), |
|
168
|
|
|
169
|
layout: getComputedStyle(header).display, |
|
170
|
|
|
171
|
height: Math.round(header.getBoundingClientRect().height) |
|
172
|
|
|
173
|
}); |
|
174
|
|
|
175
|
} |
|
176
|
|
|
177
|
// Announcement/banner bar |
|
178
|
|
|
179
|
const banner = document.querySelector('[class*="banner"], [class*="announcement"], [class*="promo-bar"], [class*="topbar"], [class*="top-bar"]'); |
|
180
|
|
|
181
|
if (banner && banner.offsetHeight > 0 && banner.offsetHeight < 100) { |
|
182
|
|
|
183
|
allSections.push({ |
|
184
|
|
|
185
|
index: 0, |
|
186
|
|
|
187
|
selector: banner.id ? '#' + banner.id : '.' + banner.className.split(' ')[0], |
|
188
|
|
|
189
|
type: 'announcement-bar', |
|
190
|
|
|
191
|
tag: banner.tagName.toLowerCase(), |
|
192
|
|
|
193
|
classes: banner.className?.toString().slice(0, 200) || '', |
|
194
|
|
|
195
|
headings: [], |
|
196
|
|
|
197
|
buttons: [], |
|
198
|
|
|
199
|
imageCount: 0, |
|
200
|
|
|
201
|
textPreview: banner.innerText.trim().slice(0, 200), |
|
202
|
|
|
203
|
layout: 'flex', |
|
204
|
|
|
205
|
height: Math.round(banner.getBoundingClientRect().height) |
|
206
|
|
|
207
|
}); |
|
208
|
|
|
209
|
} |
|
210
|
|
|
211
|
// Main content sections |
|
212
|
|
|
213
|
const children = main === document.body |
|
214
|
|
|
215
|
? [...main.children].filter(c => \!['HEADER','FOOTER','NAV','SCRIPT','STYLE','NOSCRIPT'].includes(c.tagName)) |
|
216
|
|
|
217
|
: [...main.children]; |
|
218
|
|
|
219
|
children.forEach((child, idx) => { |
|
220
|
|
|
221
|
const rect = child.getBoundingClientRect(); |
|
222
|
|
|
223
|
if (rect.height < 20) return; |
|
224
|
|
|
225
|
const cs = getComputedStyle(child); |
|
226
|
|
|
227
|
if (cs.display === 'none' || cs.visibility === 'hidden') return; |
|
228
|
|
|
229
|
const headings = [...child.querySelectorAll('h1, h2, h3, h4, h5')] |
|
230
|
|
|
231
|
.slice(0, 10).map(h => h.innerText.trim()).filter(t => t); |
|
232
|
|
|
233
|
const buttons = [...child.querySelectorAll('button, [role="button"], a[class*="btn"], a[class*="button"]')] |
|
234
|
|
|
235
|
.slice(0, 10).map(b => b.innerText.trim()).filter(t => t); |
|
236
|
|
|
237
|
const images = child.querySelectorAll('img'); |
|
238
|
|
|
239
|
const hasCarousel = child.querySelector('[class*="carousel"], [class*="slider"], [class*="swiper"], [class*="scroll"]') !== null; |
|
240
|
|
|
241
|
const hasGrid = cs.display === 'grid' || child.querySelector('[style*="grid"], [class*="grid"]') !== null; |
|
242
|
|
|
243
|
const hasFlex = cs.display === 'flex'; |
|
244
|
|
|
245
|
const hasMarquee = child.querySelector('[class*="marquee"], [class*="ticker"]') !== null; |
|
246
|
|
|
247
|
let sectionType = 'content'; |
|
248
|
|
|
249
|
if (idx === 0 && (child.querySelector('h1') || rect.height > 400)) sectionType = 'hero'; |
|
250
|
|
|
251
|
else if (hasCarousel) sectionType = 'carousel'; |
|
252
|
|
|
253
|
else if (hasMarquee) sectionType = 'marquee'; |
|
254
|
|
|
255
|
else if (hasGrid && images.length > 3) sectionType = 'grid'; |
|
256
|
|
|
257
|
else if (hasFlex && images.length === 1 && headings.length > 0) sectionType = 'split-layout'; |
|
258
|
|
|
259
|
else if (images.length > 3) sectionType = 'gallery'; |
|
260
|
|
|
261
|
allSections.push({ |
|
262
|
|
|
263
|
index: idx + 1, |
|
264
|
|
|
265
|
selector: child.id ? '#' + child.id : child.tagName.toLowerCase() + (child.className ? '.' + child.className.toString().split(' ')[0] : ''), |
|
266
|
|
|
267
|
type: sectionType, |
|
268
|
|
|
269
|
tag: child.tagName.toLowerCase(), |
|
270
|
|
|
271
|
classes: child.className?.toString().slice(0, 200) || '', |
|
272
|
|
|
273
|
headings: headings, |
|
274
|
|
|
275
|
buttons: buttons, |
|
276
|
|
|
277
|
imageCount: images.length, |
|
278
|
|
|
279
|
textPreview: child.innerText?.slice(0, 300) || '', |
|
280
|
|
|
281
|
layout: cs.display + (hasFlex ? ' ' + cs.flexDirection : '') + (hasGrid ? ' grid' : ''), |
|
282
|
|
|
283
|
height: Math.round(rect.height) |
|
284
|
|
|
285
|
}); |
|
286
|
|
|
287
|
}); |
|
288
|
|
|
289
|
// Footer |
|
290
|
|
|
291
|
const footer = document.querySelector('footer'); |
|
292
|
|
|
293
|
if (footer) { |
|
294
|
|
|
295
|
allSections.push({ |
|
296
|
|
|
297
|
index: 999, |
|
298
|
|
|
299
|
selector: 'footer', |
|
300
|
|
|
301
|
type: 'footer', |
|
302
|
|
|
303
|
tag: 'footer', |
|
304
|
|
|
305
|
classes: footer.className?.toString().slice(0, 200) || '', |
|
306
|
|
|
307
|
headings: [...footer.querySelectorAll('h3, h4, h5, strong')] |
|
308
|
|
|
309
|
.map(h => h.innerText.trim()).filter(t => t).slice(0, 10), |
|
310
|
|
|
311
|
buttons: [], |
|
312
|
|
|
313
|
imageCount: footer.querySelectorAll('img').length, |
|
314
|
|
|
315
|
textPreview: footer.innerText.trim().slice(0, 300), |
|
316
|
|
|
317
|
layout: getComputedStyle(footer).display, |
|
318
|
|
|
319
|
height: Math.round(footer.getBoundingClientRect().height) |
|
320
|
|
|
321
|
}); |
|
322
|
|
|
323
|
} |
|
324
|
|
|
325
|
return allSections; |
|
326
|
|
|
327
|
} |
|
328
|
|
|
329
|
""") |
|
330
|
|
|
331
|
# Write inventory as markdown checklist |
|
332
|
|
|
333
|
md = "# Section Inventory\\n\\n" |
|
334
|
|
|
335
|
md += "Check off each section after building and verifying it.\\n\\n" |
|
336
|
|
|
337
|
for s in inventory: |
|
338
|
|
|
339
|
md += f"- [ ] **{s['index']:02d}. {s['type'].upper()}** — `{s['selector']}`\\n" |
|
340
|
|
|
341
|
md += f" - Height: {s['height']}px | Images: {s['imageCount']} | Layout: {s['layout']}\\n" |
|
342
|
|
|
343
|
if s['headings']: |
|
344
|
|
|
345
|
md += f" - Headings: {' | '.join(s['headings'][:5])}\\n" |
|
346
|
|
|
347
|
if s['buttons']: |
|
348
|
|
|
349
|
md += f" - Buttons: {' | '.join(s['buttons'][:5])}\\n" |
|
350
|
|
|
351
|
md += f" - Text: {s['textPreview'][:150]}...\\n\\n" |
|
352
|
|
|
353
|
with open(f"{OUT_DIR}/section-inventory.md", "w") as f: |
|
354
|
|
|
355
|
f.write(md) |
|
356
|
|
|
357
|
with open(f"{OUT_DIR}/section-inventory.json", "w") as f: |
|
358
|
|
|
359
|
json.dump(inventory, f, indent=2) |
|
360
|
|
|
361
|
print(f"Inventory: {len(inventory)} sections") |
|
362
|
|
|
363
|
``` |
|
364
|
|
|
365
|
## 5. Design Token Extraction |
|
366
|
|
|
367
|
```python |
|
368
|
|
|
369
|
def extract_tokens(page, out_dir): |
|
370
|
|
|
371
|
tokens = page.evaluate(""" |
|
372
|
|
|
373
|
() => { |
|
374
|
|
|
375
|
const body = document.body; |
|
376
|
|
|
377
|
const cs = getComputedStyle(body); |
|
378
|
|
|
379
|
const cssVars = []; |
|
380
|
|
|
381
|
try { |
|
382
|
|
|
383
|
for (const sheet of document.styleSheets) { |
|
384
|
|
|
385
|
try { |
|
386
|
|
|
387
|
for (const rule of sheet.cssRules) { |
|
388
|
|
|
389
|
if (rule.selectorText === ':root' || rule.selectorText === ':root, :host') { |
|
390
|
|
|
391
|
for (const prop of rule.style) { |
|
392
|
|
|
393
|
if (prop.startsWith('--')) { |
|
394
|
|
|
395
|
cssVars.push([prop, rule.style.getPropertyValue(prop).trim()]); |
|
396
|
|
|
397
|
} |
|
398
|
|
|
399
|
} |
|
400
|
|
|
401
|
} |
|
402
|
|
|
403
|
} |
|
404
|
|
|
405
|
} catch(e) {} |
|
406
|
|
|
407
|
} |
|
408
|
|
|
409
|
} catch(e) {} |
|
410
|
|
|
411
|
const h1 = document.querySelector('h1'); |
|
412
|
|
|
413
|
const h2 = document.querySelector('h2'); |
|
414
|
|
|
415
|
const h3 = document.querySelector('h3'); |
|
416
|
|
|
417
|
const btn = document.querySelector('button, [class*="btn"], a[class*="button"]'); |
|
418
|
|
|
419
|
const nav = document.querySelector('nav, header'); |
|
420
|
|
|
421
|
const card = document.querySelector('[class*="card"], [class*="Card"]'); |
|
422
|
|
|
423
|
function getStyles(el) { |
|
424
|
|
|
425
|
if (!el) return null; |
|
426
|
|
|
427
|
const s = getComputedStyle(el); |
|
428
|
|
|
429
|
return { |
|
430
|
|
|
431
|
fontSize: s.fontSize, fontWeight: s.fontWeight, fontFamily: s.fontFamily, |
|
432
|
|
|
433
|
lineHeight: s.lineHeight, letterSpacing: s.letterSpacing, color: s.color, |
|
434
|
|
|
435
|
textTransform: s.textTransform, textAlign: s.textAlign, |
|
436
|
|
|
437
|
backgroundColor: s.backgroundColor, |
|
438
|
|
|
439
|
padding: s.padding, borderRadius: s.borderRadius, border: s.border, |
|
440
|
|
|
441
|
boxShadow: s.boxShadow |
|
442
|
|
|
443
|
}; |
|
444
|
|
|
445
|
} |
|
446
|
|
|
447
|
return { |
|
448
|
|
|
449
|
body: { |
|
450
|
|
|
451
|
bgColor: cs.backgroundColor, textColor: cs.color, |
|
452
|
|
|
453
|
fontFamily: cs.fontFamily, fontSize: cs.fontSize, lineHeight: cs.lineHeight |
|
454
|
|
|
455
|
}, |
|
456
|
|
|
457
|
h1: getStyles(h1), h2: getStyles(h2), h3: getStyles(h3), |
|
458
|
|
|
459
|
button: getStyles(btn), nav: getStyles(nav), card: getStyles(card), |
|
460
|
|
|
461
|
cssVars: cssVars, |
|
462
|
|
|
463
|
fonts: [...document.querySelectorAll('link[href*="fonts.googleapis"], link[href*="fonts.gstatic"]')] |
|
464
|
|
|
465
|
.map(l => l.href), |
|
466
|
|
|
467
|
selfHostedFonts: (() => { |
|
468
|
|
|
469
|
const fonts = []; |
|
470
|
|
|
471
|
try { |
|
472
|
|
|
473
|
for (const sheet of document.styleSheets) { |
|
474
|
|
|
475
|
try { |
|
476
|
|
|
477
|
for (const rule of sheet.cssRules) { |
|
478
|
|
|
479
|
if (rule instanceof CSSFontFaceRule) { |
|
480
|
|
|
481
|
fonts.push({ |
|
482
|
|
|
483
|
family: rule.style.getPropertyValue('font-family').replace(/['"]/g, ''), |
|
484
|
|
|
485
|
src: rule.style.getPropertyValue('src'), |
|
486
|
|
|
487
|
weight: rule.style.getPropertyValue('font-weight') || '400', |
|
488
|
|
|
489
|
style: rule.style.getPropertyValue('font-style') || 'normal' |
|
490
|
|
|
491
|
}); |
|
492
|
|
|
493
|
} |
|
494
|
|
|
495
|
} |
|
496
|
|
|
497
|
} catch(e) {} |
|
498
|
|
|
499
|
} |
|
500
|
|
|
501
|
} catch(e) {} |
|
502
|
|
|
503
|
return fonts; |
|
504
|
|
|
505
|
})(), |
|
506
|
|
|
507
|
favicons: [...document.querySelectorAll('link[rel*="icon"], link[rel="apple-touch-icon"]')] |
|
508
|
|
|
509
|
.map(l => ({ href: l.href, rel: l.rel, sizes: l.sizes?.toString() || '' })), |
|
510
|
|
|
511
|
title: document.title, |
|
512
|
|
|
513
|
metaDescription: document.querySelector('meta[name="description"]')?.content || '' |
|
514
|
|
|
515
|
}; |
|
516
|
|
|
517
|
} |
|
518
|
|
|
519
|
""") |
|
520
|
|
|
521
|
with open(f"{out_dir}/tokens.json", "w") as f: |
|
522
|
|
|
523
|
json.dump(tokens, f, indent=2) |
|
524
|
|
|
525
|
return tokens |
|
526
|
|
|
527
|
``` |
|
528
|
|
|
529
|
## 6. Asset Download (Complete) |
|
530
|
|
|
531
|
```python |
|
532
|
|
|
533
|
def download_all_assets(page, out_dir="clone-data", public_dir="public"): |
|
534
|
|
|
535
|
assets = page.evaluate(""" |
|
536
|
|
|
537
|
() => { |
|
538
|
|
|
539
|
return { |
|
540
|
|
|
541
|
images: [...document.querySelectorAll('img')] |
|
542
|
|
|
543
|
.filter(img => img.offsetWidth > 30 && img.src) |
|
544
|
|
|
545
|
.map(img => ({ |
|
546
|
|
|
547
|
src: img.src, |
|
548
|
|
|
549
|
srcset: img.srcset || '', |
|
550
|
|
|
551
|
alt: img.alt, |
|
552
|
|
|
553
|
w: img.offsetWidth, h: img.offsetHeight, |
|
554
|
|
|
555
|
parentClasses: img.parentElement?.className?.toString().slice(0, 100) || '' |
|
556
|
|
|
557
|
})), |
|
558
|
|
|
559
|
videos: [...document.querySelectorAll('video')].map(v => ({ |
|
560
|
|
|
561
|
src: v.src || v.querySelector('source')?.src, |
|
562
|
|
|
563
|
poster: v.poster, |
|
564
|
|
|
565
|
autoplay: v.autoplay, loop: v.loop, muted: v.muted |
|
566
|
|
|
567
|
})).filter(v => v.src), |
|
568
|
|
|
569
|
backgroundImages: [...document.querySelectorAll('*')].filter(el => { |
|
570
|
|
|
571
|
const bg = getComputedStyle(el).backgroundImage; |
|
572
|
|
|
573
|
return bg && bg !== 'none' && bg.includes('url('); |
|
574
|
|
|
575
|
}).slice(0, 50).map(el => ({ |
|
576
|
|
|
577
|
url: getComputedStyle(el).backgroundImage, |
|
578
|
|
|
579
|
element: el.tagName + (el.className ? '.' + el.className.toString().split(' ')[0] : '') |
|
580
|
|
|
581
|
})), |
|
582
|
|
|
583
|
svgs: [...document.querySelectorAll('svg')].slice(0, 50).map((svg, i) => ({ |
|
584
|
|
|
585
|
index: i, |
|
586
|
|
|
587
|
viewBox: svg.getAttribute('viewBox') || '', |
|
588
|
|
|
589
|
width: svg.getAttribute('width') || svg.offsetWidth, |
|
590
|
|
|
591
|
height: svg.getAttribute('height') || svg.offsetHeight, |
|
592
|
|
|
593
|
html: svg.outerHTML.length < 5000 ? svg.outerHTML : '[TOO_LARGE]', |
|
594
|
|
|
595
|
parentText: svg.parentElement?.innerText?.trim().slice(0, 50) || '', |
|
596
|
|
|
597
|
ariaLabel: svg.getAttribute('aria-label') || '' |
|
598
|
|
|
599
|
})) |
|
600
|
|
|
601
|
}; |
|
602
|
|
|
603
|
} |
|
604
|
|
|
605
|
""") |
|
606
|
|
|
607
|
with open(f"{out_dir}/assets.json", "w") as f: |
|
608
|
|
|
609
|
json.dump(assets, f, indent=2) |
|
610
|
|
|
611
|
downloaded = {} |
|
612
|
|
|
613
|
img_dir = f"{public_dir}/images" |
|
614
|
|
|
615
|
vid_dir = f"{public_dir}/videos" |
|
616
|
|
|
617
|
os.makedirs(img_dir, exist_ok=True) |
|
618
|
|
|
619
|
os.makedirs(vid_dir, exist_ok=True) |
|
620
|
|
|
621
|
def upscale_cdn_url(url): |
|
622
|
|
|
623
|
"""Increase resolution for known CDN URL patterns.""" |
|
624
|
|
|
625
|
if 'cdn.shopify.com' in url or 'aloyoga.com/cdn' in url: |
|
626
|
|
|
627
|
url = re.sub(r'_small\', '_1200x.', url) |
|
628
|
|
|
629
|
url = re.sub(r'width=\d+', 'width=1200', url) |
|
630
|
|
|
631
|
url = re.sub(r'height=\d+', 'height=1200', url) |
|
632
|
|
|
633
|
elif 'cdn.sanity.io' in url: |
|
634
|
|
|
635
|
url = re.sub(r'w=\d+', 'w=1200', url) |
|
636
|
|
|
637
|
url = re.sub(r'h=\d+', 'h=1200', url) |
|
638
|
|
|
639
|
elif 'res.cloudinary.com' in url: |
|
640
|
|
|
641
|
url = re.sub(r'w_\d+', 'w_1200', url) |
|
642
|
|
|
643
|
url = re.sub(r'h_\d+', 'h_1200', url) |
|
644
|
|
|
645
|
elif 'images.ctfassets.net' in url: |
|
646
|
|
|
647
|
url = re.sub(r'w=\d+', 'w=1200', url) |
|
648
|
|
|
649
|
return url |
|
650
|
|
|
651
|
def download_with_retry(url, filepath, max_retries=3): |
|
652
|
|
|
653
|
"""Download a URL with retry logic and fallback User-Agent strings.""" |
|
654
|
|
|
655
|
user_agents = [ |
|
656
|
|
|
657
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", |
|
658
|
|
|
659
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", |
|
660
|
|
|
661
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", |
|
662
|
|
|
663
|
] |
|
664
|
|
|
665
|
last_error = None |
|
666
|
|
|
667
|
for attempt in range(max_retries): |
|
668
|
|
|
669
|
try: |
|
670
|
|
|
671
|
ua = user_agents[attempt % len(user_agents)] |
|
672
|
|
|
673
|
req = urllib.request.Request(url, headers={"User-Agent": ua, "Accept": "image/*,*/*"}) |
|
674
|
|
|
675
|
with urllib.request.urlopen(req, timeout=15) as resp: |
|
676
|
|
|
677
|
status = resp.getcode() |
|
678
|
|
|
679
|
if status != 200: |
|
680
|
|
|
681
|
print(f" Attempt {attempt+1}: HTTP {status} for {url}") |
|
682
|
|
|
683
|
last_error = f"HTTP {status}" |
|
684
|
|
|
685
|
continue |
|
686
|
|
|
687
|
data = resp.read() |
|
688
|
|
|
689
|
if len(data) < 100: |
|
690
|
|
|
691
|
print(f" Attempt {attempt+1}: Suspiciously small ({len(data)} bytes)") |
|
692
|
|
|
693
|
last_error = f"Too small ({len(data)} bytes)" |
|
694
|
|
|
695
|
continue |
|
696
|
|
|
697
|
with open(filepath, "wb") as f: |
|
698
|
|
|
699
|
f.write(data) |
|
700
|
|
|
701
|
return True |
|
702
|
|
|
703
|
except Exception as e: |
|
704
|
|
|
705
|
last_error = str(e) |
|
706
|
|
|
707
|
print(f" Attempt {attempt+1} failed: {e}") |
|
708
|
|
|
709
|
print(f"FAIL after {max_retries} attempts: {url} -> {last_error}") |
|
710
|
|
|
711
|
return False |
|
712
|
|
|
713
|
for img in assets.get("images", []): |
|
714
|
|
|
715
|
url = img["src"] |
|
716
|
|
|
717
|
if not url or url.startswith("data:"): |
|
718
|
|
|
719
|
continue |
|
720
|
|
|
721
|
url = upscale_cdn_url(url) |
|
722
|
|
|
723
|
ext = re.search(r'\(png|jpg|jpeg|webp|gif|svg|avif)', url.lower()) |
|
724
|
|
|
725
|
ext = ext.group(0) if ext else ".webp" |
|
726
|
|
|
727
|
name_hash = hashlib.md5(url.encode()).hexdigest()[:10] |
|
728
|
|
|
729
|
alt_slug = re.sub(r'[^a-z0-9]', '-', (img.get("alt") or "img").lower())[:30] |
|
730
|
|
|
731
|
filename = f"{alt_slug}-{name_hash}{ext}" |
|
732
|
|
|
733
|
filepath = f"{img_dir}/{filename}" |
|
734
|
|
|
735
|
if download_with_retry(url, filepath): |
|
736
|
|
|
737
|
downloaded[img["src"]] = f"/images/{filename}" |
|
738
|
|
|
739
|
print(f"OK: {filename}") |
|
740
|
|
|
741
|
else: |
|
742
|
|
|
743
|
no_upscale = img["src"] |
|
744
|
|
|
745
|
if no_upscale != url: |
|
746
|
|
|
747
|
print(f" Retrying without CDN upscale: {no_upscale}") |
|
748
|
|
|
749
|
if download_with_retry(no_upscale, filepath): |
|
750
|
|
|
751
|
downloaded[img["src"]] = f"/images/{filename}" |
|
752
|
|
|
753
|
print(f"OK (no upscale): {filename}") |
|
754
|
|
|
755
|
for vid in assets.get("videos", []): |
|
756
|
|
|
757
|
url = vid["src"] |
|
758
|
|
|
759
|
if not url: continue |
|
760
|
|
|
761
|
ext = re.search(r'\(mp4|webm|mov)', url.lower()) |
|
762
|
|
|
763
|
ext = ext.group(0) if ext else ".mp4" |
|
764
|
|
|
765
|
name_hash = hashlib.md5(url.encode()).hexdigest()[:10] |
|
766
|
|
|
767
|
filename = f"video-{name_hash}{ext}" |
|
768
|
|
|
769
|
filepath = f"{vid_dir}/{filename}" |
|
770
|
|
|
771
|
try: |
|
772
|
|
|
773
|
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) |
|
774
|
|
|
775
|
with urllib.request.urlopen(req, timeout=30) as resp: |
|
776
|
|
|
777
|
with open(filepath, "wb") as f: |
|
778
|
|
|
779
|
f.write(resp.read()) |
|
780
|
|
|
781
|
downloaded[url] = f"/videos/{filename}" |
|
782
|
|
|
783
|
print(f"OK: {filename}") |
|
784
|
|
|
785
|
except Exception as e: |
|
786
|
|
|
787
|
print(f"FAIL: {url} -> {e}") |
|
788
|
|
|
789
|
with open(f"{out_dir}/downloaded-assets.json", "w") as f: |
|
790
|
|
|
791
|
json.dump(downloaded, f, indent=2) |
|
792
|
|
|
793
|
# Verify all downloads |
|
794
|
|
|
795
|
failed = [] |
|
796
|
|
|
797
|
for original_url, local_path in downloaded.items(): |
|
798
|
|
|
799
|
full_path = f"{public_dir}{local_path}" |
|
800
|
|
|
801
|
if not os.path.exists(full_path) or os.path.getsize(full_path) < 100: |
|
802
|
|
|
803
|
failed.append(original_url) |
|
804
|
|
|
805
|
if failed: |
|
806
|
|
|
807
|
print(f"\\nWARNING: {len(failed)} assets failed verification:") |
|
808
|
|
|
809
|
for url in failed: |
|
810
|
|
|
811
|
print(f" - {url}") |
|
812
|
|
|
813
|
return downloaded |
|
814
|
|
|
815
|
``` |
|
816
|
|
|
817
|
## 7. Per-Component Deep Style Extraction |
|
818
|
|
|
819
|
```python |
|
820
|
|
|
821
|
def extract_component_styles(page, selector): |
|
822
|
|
|
823
|
return page.evaluate(""" |
|
824
|
|
|
825
|
(selector) => { |
|
826
|
|
|
827
|
const el = document.querySelector(selector); |
|
828
|
|
|
829
|
if (!el) return { error: 'Element not found: ' + selector }; |
|
830
|
|
|
831
|
const props = [ |
|
832
|
|
|
833
|
'fontSize','fontWeight','fontFamily','lineHeight','letterSpacing','color', |
|
834
|
|
|
835
|
'textTransform','textDecoration','textAlign', |
|
836
|
|
|
837
|
'backgroundColor','background','backgroundImage','backgroundSize','backgroundPosition', |
|
838
|
|
|
839
|
'padding','paddingTop','paddingRight','paddingBottom','paddingLeft', |
|
840
|
|
|
841
|
'margin','marginTop','marginRight','marginBottom','marginLeft', |
|
842
|
|
|
843
|
'width','height','maxWidth','minWidth','maxHeight','minHeight', |
|
844
|
|
|
845
|
'display','flexDirection','justifyContent','alignItems','gap', |
|
846
|
|
|
847
|
'gridTemplateColumns','gridTemplateRows', |
|
848
|
|
|
849
|
'borderRadius','border','borderTop','borderBottom','borderLeft','borderRight', |
|
850
|
|
|
851
|
'boxShadow','overflow','overflowX','overflowY', |
|
852
|
|
|
853
|
'position','top','right','bottom','left','zIndex', |
|
854
|
|
|
855
|
'opacity','transform','transition','cursor', |
|
856
|
|
|
857
|
'objectFit','objectPosition','whiteSpace','textOverflow' |
|
858
|
|
|
859
|
]; |
|
860
|
|
|
861
|
function extractStyles(element) { |
|
862
|
|
|
863
|
const cs = getComputedStyle(element); |
|
864
|
|
|
865
|
const styles = {}; |
|
866
|
|
|
867
|
props.forEach(p => { |
|
868
|
|
|
869
|
const v = cs[p]; |
|
870
|
|
|
871
|
if (v && v !== 'none' && v !== 'normal' && v !== 'auto' && v !== '0px' && |
|
872
|
|
|
873
|
v !== 'rgba(0, 0, 0, 0)' && v !== 'start' && v !== 'stretch' && |
|
874
|
|
|
875
|
v !== 'visible' && v !== 'static' && v !== 'row' && v !== 'repeat') { |
|
876
|
|
|
877
|
styles[p] = v; |
|
878
|
|
|
879
|
} |
|
880
|
|
|
881
|
}); |
|
882
|
|
|
883
|
return styles; |
|
884
|
|
|
885
|
} |
|
886
|
|
|
887
|
function walk(element, depth) { |
|
888
|
|
|
889
|
if (depth > 4) return null; |
|
890
|
|
|
891
|
const children = [...element.children]; |
|
892
|
|
|
893
|
const rect = element.getBoundingClientRect(); |
|
894
|
|
|
895
|
return { |
|
896
|
|
|
897
|
tag: element.tagName.toLowerCase(), |
|
898
|
|
|
899
|
classes: element.className?.toString().split(' ').slice(0, 5).join(' ') || '', |
|
900
|
|
|
901
|
id: element.id || null, |
|
902
|
|
|
903
|
text: element.childNodes.length === 1 && element.childNodes[0].nodeType === 3 |
|
904
|
|
|
905
|
? element.textContent.trim().slice(0, 300) : null, |
|
906
|
|
|
907
|
rect: { x: Math.round(rect.x), y: Math.round(rect.y), |
|
908
|
|
|
909
|
w: Math.round(rect.width), h: Math.round(rect.height) }, |
|
910
|
|
|
911
|
styles: extractStyles(element), |
|
912
|
|
|
913
|
images: element.tagName === 'IMG' ? { |
|
914
|
|
|
915
|
src: element.src, alt: element.alt, |
|
916
|
|
|
917
|
naturalWidth: element.naturalWidth, naturalHeight: element.naturalHeight |
|
918
|
|
|
919
|
} : null, |
|
920
|
|
|
921
|
svg: element.tagName === 'SVG' ? { |
|
922
|
|
|
923
|
viewBox: element.getAttribute('viewBox'), |
|
924
|
|
|
925
|
html: element.outerHTML.length < 3000 ? element.outerHTML : '[TOO_LARGE]' |
|
926
|
|
|
927
|
} : null, |
|
928
|
|
|
929
|
childCount: children.length, |
|
930
|
|
|
931
|
children: children.slice(0, 20).map(c => walk(c, depth + 1)).filter(Boolean) |
|
932
|
|
|
933
|
}; |
|
934
|
|
|
935
|
} |
|
936
|
|
|
937
|
return walk(el, 0); |
|
938
|
|
|
939
|
} |
|
940
|
|
|
941
|
""", selector) |
|
942
|
|
|
943
|
``` |
|
944
|
|
|
945
|
## 8. Hover and Scroll State Extraction |
|
946
|
|
|
947
|
```python |
|
948
|
|
|
949
|
def extract_hover_state(page, selector): |
|
950
|
|
|
951
|
before = page.evaluate(""" |
|
952
|
|
|
953
|
(sel) => { |
|
954
|
|
|
955
|
const el = document.querySelector(sel); |
|
956
|
|
|
957
|
if (!el) return null; |
|
958
|
|
|
959
|
const cs = getComputedStyle(el); |
|
960
|
|
|
961
|
return { |
|
962
|
|
|
963
|
backgroundColor: cs.backgroundColor, color: cs.color, |
|
964
|
|
|
965
|
transform: cs.transform, boxShadow: cs.boxShadow, |
|
966
|
|
|
967
|
opacity: cs.opacity, borderColor: cs.borderColor, |
|
968
|
|
|
969
|
textDecoration: cs.textDecoration, transition: cs.transition |
|
970
|
|
|
971
|
}; |
|
972
|
|
|
973
|
} |
|
974
|
|
|
975
|
""", selector) |
|
976
|
|
|
977
|
el = page.query_selector(selector) |
|
978
|
|
|
979
|
if el: |
|
980
|
|
|
981
|
el.hover() |
|
982
|
|
|
983
|
page.wait_for_timeout(500) |
|
984
|
|
|
985
|
after = page.evaluate(""" |
|
986
|
|
|
987
|
(sel) => { |
|
988
|
|
|
989
|
const el = document.querySelector(sel); |
|
990
|
|
|
991
|
if (!el) return null; |
|
992
|
|
|
993
|
const cs = getComputedStyle(el); |
|
994
|
|
|
995
|
return { |
|
996
|
|
|
997
|
backgroundColor: cs.backgroundColor, color: cs.color, |
|
998
|
|
|
999
|
transform: cs.transform, boxShadow: cs.boxShadow, |
|
1000
|
|
|
1001
|
opacity: cs.opacity, borderColor: cs.borderColor, |
|
1002
|
|
|
1003
|
textDecoration: cs.textDecoration, transition: cs.transition |
|
1004
|
|
|
1005
|
}; |
|
1006
|
|
|
1007
|
} |
|
1008
|
|
|
1009
|
""", selector) |
|
1010
|
|
|
1011
|
diff = {} |
|
1012
|
|
|
1013
|
if before and after: |
|
1014
|
|
|
1015
|
for key in before: |
|
1016
|
|
|
1017
|
if before[key] != after[key]: |
|
1018
|
|
|
1019
|
diff[key] = {"before": before[key], "after": after[key]} |
|
1020
|
|
|
1021
|
return {"before": before, "after": after, "diff": diff} |
|
1022
|
|
|
1023
|
def extract_scroll_state(page, selector, scroll_to=200): |
|
1024
|
|
|
1025
|
page.evaluate("window.scrollTo(0, 0)") |
|
1026
|
|
|
1027
|
page.wait_for_timeout(500) |
|
1028
|
|
|
1029
|
before = page.evaluate(""" |
|
1030
|
|
|
1031
|
(sel) => { |
|
1032
|
|
|
1033
|
const el = document.querySelector(sel); |
|
1034
|
|
|
1035
|
if (!el) return null; |
|
1036
|
|
|
1037
|
const cs = getComputedStyle(el); |
|
1038
|
|
|
1039
|
return { |
|
1040
|
|
|
1041
|
backgroundColor: cs.backgroundColor, boxShadow: cs.boxShadow, |
|
1042
|
|
|
1043
|
height: cs.height, padding: cs.padding, borderRadius: cs.borderRadius, |
|
1044
|
|
|
1045
|
position: cs.position, top: cs.top, transform: cs.transform, |
|
1046
|
|
|
1047
|
opacity: cs.opacity, backdropFilter: cs.backdropFilter |
|
1048
|
|
|
1049
|
}; |
|
1050
|
|
|
1051
|
} |
|
1052
|
|
|
1053
|
""", selector) |
|
1054
|
|
|
1055
|
page.evaluate(f"window.scrollTo(0, {scroll_to})") |
|
1056
|
|
|
1057
|
page.wait_for_timeout(800) |
|
1058
|
|
|
1059
|
after = page.evaluate(""" |
|
1060
|
|
|
1061
|
(sel) => { |
|
1062
|
|
|
1063
|
const el = document.querySelector(sel); |
|
1064
|
|
|
1065
|
if (!el) return null; |
|
1066
|
|
|
1067
|
const cs = getComputedStyle(el); |
|
1068
|
|
|
1069
|
return { |
|
1070
|
|
|
1071
|
backgroundColor: cs.backgroundColor, boxShadow: cs.boxShadow, |
|
1072
|
|
|
1073
|
height: cs.height, padding: cs.padding, borderRadius: cs.borderRadius, |
|
1074
|
|
|
1075
|
position: cs.position, top: cs.top, transform: cs.transform, |
|
1076
|
|
|
1077
|
opacity: cs.opacity, backdropFilter: cs.backdropFilter |
|
1078
|
|
|
1079
|
}; |
|
1080
|
|
|
1081
|
} |
|
1082
|
|
|
1083
|
""", selector) |
|
1084
|
|
|
1085
|
page.evaluate("window.scrollTo(0, 0)") |
|
1086
|
|
|
1087
|
diff = {} |
|
1088
|
|
|
1089
|
if before and after: |
|
1090
|
|
|
1091
|
for key in before: |
|
1092
|
|
|
1093
|
if before[key] != after[key]: |
|
1094
|
|
|
1095
|
diff[key] = {"before": before[key], "after": after[key]} |
|
1096
|
|
|
1097
|
return {"scrollThreshold": scroll_to, "before": before, "after": after, "diff": diff} |
|
1098
|
|
|
1099
|
``` |
|
1100
|
|
|
1101
|
## 9. Responsive Layout Extraction |
|
1102
|
|
|
1103
|
```python |
|
1104
|
|
|
1105
|
def extract_responsive(page, selector, breakpoints=[1440, 768, 390]): |
|
1106
|
|
|
1107
|
results = {} |
|
1108
|
|
|
1109
|
for width in breakpoints: |
|
1110
|
|
|
1111
|
page.set_viewport_size({"width": width, "height": 900}) |
|
1112
|
|
|
1113
|
page.wait_for_timeout(800) |
|
1114
|
|
|
1115
|
data = page.evaluate(""" |
|
1116
|
|
|
1117
|
(sel) => { |
|
1118
|
|
|
1119
|
const el = document.querySelector(sel); |
|
1120
|
|
|
1121
|
if (!el) return null; |
|
1122
|
|
|
1123
|
const cs = getComputedStyle(el); |
|
1124
|
|
|
1125
|
const rect = el.getBoundingClientRect(); |
|
1126
|
|
|
1127
|
return { |
|
1128
|
|
|
1129
|
display: cs.display, flexDirection: cs.flexDirection, |
|
1130
|
|
|
1131
|
gridTemplateColumns: cs.gridTemplateColumns, gap: cs.gap, |
|
1132
|
|
|
1133
|
padding: cs.padding, width: Math.round(rect.width), |
|
1134
|
|
|
1135
|
height: Math.round(rect.height), childCount: el.children.length, |
|
1136
|
|
|
1137
|
childrenVisible: [...el.children].filter(c => { |
|
1138
|
|
|
1139
|
const s = getComputedStyle(c); |
|
1140
|
|
|
1141
|
return s.display !== 'none' && s.visibility !== 'hidden'; |
|
1142
|
|
|
1143
|
}).length |
|
1144
|
|
|
1145
|
}; |
|
1146
|
|
|
1147
|
} |
|
1148
|
|
|
1149
|
""", selector) |
|
1150
|
|
|
1151
|
results[f"{width}px"] = data |
|
1152
|
|
|
1153
|
page.set_viewport_size({"width": 1440, "height": 900}) |
|
1154
|
|
|
1155
|
return results |
|
1156
|
|
|
1157
|
``` |
|
1158
|
|
|
1159
|
## 10. Complete Per-Section Extraction Flow |
|
1160
|
|
|
1161
|
```python |
|
1162
|
|
|
1163
|
def extract_section_full(page, section_name, selector, out_dir="clone-data"): |
|
1164
|
|
|
1165
|
print(f"\\nExtracting: {section_name} ({selector})") |
|
1166
|
|
|
1167
|
el = page.query_selector(selector) |
|
1168
|
|
|
1169
|
if el: |
|
1170
|
|
|
1171
|
el.scroll_into_view_if_needed() |
|
1172
|
|
|
1173
|
page.wait_for_timeout(500) |
|
1174
|
|
|
1175
|
el.screenshot(path=f"{out_dir}/screenshots/{section_name}.png") |
|
1176
|
|
|
1177
|
styles = extract_component_styles(page, selector) |
|
1178
|
|
|
1179
|
with open(f"{out_dir}/components/{section_name}-styles.json", "w") as f: |
|
1180
|
|
|
1181
|
json.dump(styles, f, indent=2) |
|
1182
|
|
|
1183
|
responsive = extract_responsive(page, selector) |
|
1184
|
|
|
1185
|
with open(f"{out_dir}/components/{section_name}-responsive.json", "w") as f: |
|
1186
|
|
|
1187
|
json.dump(responsive, f, indent=2) |
|
1188
|
|
|
1189
|
text = page.evaluate(""" |
|
1190
|
|
|
1191
|
(sel) => { |
|
1192
|
|
|
1193
|
const el = document.querySelector(sel); |
|
1194
|
|
|
1195
|
return el ? el.innerText : ''; |
|
1196
|
|
|
1197
|
} |
|
1198
|
|
|
1199
|
""", selector) |
|
1200
|
|
|
1201
|
with open(f"{out_dir}/components/{section_name}-text.txt", "w") as f: |
|
1202
|
|
|
1203
|
f.write(text) |
|
1204
|
|
|
1205
|
return {"styles": styles, "responsive": responsive, "text": text} |
|
1206
|
|
|
1207
|
``` |