Robot Txt Generator

 

def generate_robots_txt(): # Define the user agents and their rules user_agents = { 'Googlebot': ['/allow-me/', '/allow-me-too/', '/disallow-me-not/'], 'Bingbot': ['/allow-me/', '/allow-me-too/', '/disallow-me-not/'], # Add more user agents and rules as needed } # Create the robots.txt content content = "User-agent: *\n" # Add rules for each user agent for agent, rules in user_agents.items(): content += f"User-agent: {agent}\n" for rule in rules: content += f"Disallow: {rule}\n" # Save the robots.txt file with open('robots.txt', 'w') as file: file.write(content) print("robots.txt generated successfully!") if __name__ == "__main__": generate_robots_txt()