User-agent: Scrapy
Allow: /

sitemap: https://www.pmfst.unist.hr/sitemap.xml



User-agent:  *
# disallow all files in these directories
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /archives/
Disallow: /comments/feed/